diff --git a/.gitignore b/.gitignore index 9b6ccbf..209586a 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,11 @@ data/agent_roster.json # Local scratch notes .notes/ +# Superpowers working files (implementation plans, per-session scratch). Local +# process artifacts, not project documentation — docs/agent-flow.md and specs/ +# are where anything durable belongs. +docs/superpowers/ + # Slack provisioning state — holds app client_secrets during a bulk provisioning # run (scripts/provision_slack_bots.py). Must never be committed. .provision_state.json @@ -75,6 +80,12 @@ data/agent_roster.json .mutmut-cache mutants/ +# Playwright MCP browser artifacts (console logs, page snapshots) +.playwright-mcp/ + +# Production migration dumps (scripts/migrate/run_migration.sh). Never commit these: +# they are full database copies containing message bodies and tokens. +backups/ # Local Claude Code state .claude/ diff --git a/CLAUDE.md b/CLAUDE.md index 922a602..c19aeca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,9 +2,26 @@ ## Testing -Run `python -m pytest tests/ -v` before committing. All tests must pass. -Tests run inside Docker: `docker compose exec app python -m pytest tests/ -v` -(may need `pip install pytest pytest-asyncio` first if the container was rebuilt). +Run `./scripts/ci.sh` before committing — alembic sanity (single head, no +duplicate revision ids), `ruff check` on the test suite, then the full pytest run +with a branch-coverage floor. This is exactly what the `pre-push` hook runs, and +it is the whole gate: there is no server-side CI. + +To run pytest alone **inside the container**, `TEST_DATABASE_URL` is required. +Without it `tests/conftest.py` falls back to spinning an ephemeral Postgres via +testcontainers, and the `app` container has no Docker socket — so every test that +needs a database errors out (469 of them, measured 2026-08-04): + +```bash +docker compose exec -T -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + app python -m pytest tests/ -v +``` + +The named database must already exist — the suite migrates it, it does not create +it. Add a fresh scratch DB with +`docker compose exec -T postgres createdb -U copi copi_xN`, and give concurrent +suites distinct names so they do not migrate each other's schema mid-run. Never +point `TEST_DATABASE_URL` at `copi`, the dev database. ## Compose file set (read this before any `docker compose` command) @@ -68,8 +85,12 @@ C="-f docker-compose.prod.yml -f docker-compose.override.yml" docker logs agent-run > logs/run_$(date +%s).log 2>&1 ls -t logs/run_*.log | tail -n +11 | xargs rm -f -# 2. Stop the old container -docker rm -f agent-run +# 2. Stop the old container — GRACEFULLY. `docker rm -f` sends SIGKILL, which +# skips the shutdown flush and permanently loses the in-flight turn's +# messages (the DB, not Slack, is the durable store). `docker stop` sends +# SIGTERM; -t 30 leaves room for an in-flight LLM call to finish. +docker stop -t 30 agent-run +docker rm agent-run # 3. Rebuild app + worker (picks up code changes) docker compose $C up -d --build app worker @@ -135,14 +156,14 @@ rotating pair is persisted in the `app_settings` KV table) and a public `base_ur roster from the container, then run the script on the host: ```bash -docker exec copi-python-app-1 python scripts/export_agent_roster.py # writes data/agent_roster.json +docker compose exec app python scripts/export_agent_roster.py # writes data/agent_roster.json python3 scripts/provision_slack_bots.py # host: creates apps, prints OAuth URLs ``` The host script writes tokens to `.env`; import them into the DB column with: ```bash -docker exec copi-python-app-1 python scripts/backfill_agent_tokens.py +docker compose exec app python scripts/backfill_agent_tokens.py ``` (`.env` + `config.py get_slack_tokens()` remain a read fallback, but the DB column is diff --git a/README.md b/README.md index ec1094d..c4a7fe3 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,14 @@ Cross-cutting: ```bash cp .env.example .env # fill in Anthropic, Slack, ORCID, SMTP credentials docker compose up -d --build app worker postgres + +# Migrate. Check for a single head FIRST: two migrations sharing a revision id +# (a stale branch renumbered late) makes `upgrade head` fail on multiple heads, +# and makes a targeted `upgrade ` silently skip one of them while stamping +# the DB as fully migrated. `alembic heads` needs no database. +docker compose exec app alembic heads # must print exactly one line docker compose exec app alembic upgrade head +docker compose exec app alembic current # confirm it advanced ``` Web UI: . @@ -71,7 +78,8 @@ Before restarting, save logs and rebuild: ```bash docker logs agent-run > logs/run_$(date +%s).log 2>&1 ls -t logs/run_*.log | tail -n +11 | xargs rm -f -docker rm -f agent-run +docker stop -t 30 agent-run # SIGTERM: lets the engine flush before exit +docker rm agent-run docker compose up -d --build app worker docker compose --profile agent run -d --name agent-run agent \ python -m src.agent.main --budget 0 diff --git a/alembic/env.py b/alembic/env.py index 1d1f037..925f458 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -42,6 +42,39 @@ def run_migrations_offline() -> None: context.run_migrations() +#: How long a migration will WAIT for a lock before giving up, in milliseconds. +#: 0 disables the bound (Postgres' own default, and what this file did before). +#: +#: Why this exists. `context.configure()` below is deliberately NOT passed +#: `transaction_per_migration`, so the entire upgrade chain runs in ONE +#: transaction. That is good — a killed migration cannot leave a half-applied +#: schema, verified by terminating the backend mid-chain. But it means every lock +#: the chain takes is held until the final commit, and migration 0019 takes +#: ACCESS EXCLUSIVE on `agent_messages` to add three indexes and a unique +#: constraint (which brings a fourth index of its own). +#: +#: With no lock_timeout, `alembic upgrade` parked behind a single open +#: `BEGIN; SELECT …` waits forever — and because a pending ACCESS EXCLUSIVE +#: request queues ahead of new readers, every subsequent query on that table +#: blocks behind it. One forgotten transaction plus a migration is a total stall +#: on the hot table, with no timeout to end it. Failing fast and retrying in a +#: quieter moment is strictly better than an unbounded outage: the transaction +#: rolls back cleanly, so a timeout costs nothing but the attempt. +#: +#: This bounds only the WAIT for a lock. It is not `statement_timeout`, which +#: would cancel a legitimately long index build partway through. +LOCK_TIMEOUT_MS = os.environ.get("ALEMBIC_LOCK_TIMEOUT_MS", "10000") + + +#: NOTE ON HOW THIS IS APPLIED. It is set as an asyncpg *connect* setting, not by +#: executing `SET lock_timeout = …` on the connection inside do_run_migrations(). +#: The obvious version of that is quietly catastrophic: `connection.exec_driver_sql` +#: before `context.begin_transaction()` opens its own transaction, alembic's +#: transaction then nests inside it, and the outer `async with connect()` exits +#: without committing — so every migration LOGS "Running upgrade" and the whole +#: chain SILENTLY ROLLS BACK, leaving no `alembic_version` table at all. Observed +#: while writing this: 18 migrations "applied", nothing persisted. Counting the log +#: lines is not a verification; always re-read `alembic_version` afterwards. def do_run_migrations(connection: Connection) -> None: context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): @@ -50,11 +83,15 @@ def do_run_migrations(connection: Connection) -> None: async def run_async_migrations() -> None: """In this scenario we need to create an Engine and associate a connection with the context.""" - connectable = async_engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) + cfg = config.get_section(config.config_ini_section, {}) + kwargs: dict = {"prefix": "sqlalchemy.", "poolclass": pool.NullPool} + if LOCK_TIMEOUT_MS and LOCK_TIMEOUT_MS != "0": + # asyncpg takes libpq-style GUCs via server_settings, applied at connect + # time — outside any transaction, so it cannot disturb alembic's. + kwargs["connect_args"] = { + "server_settings": {"lock_timeout": str(int(LOCK_TIMEOUT_MS))} + } + connectable = async_engine_from_config(cfg, **kwargs) async with connectable.connect() as connection: await connection.run_sync(do_run_migrations) await connectable.dispose() diff --git a/alembic/versions/0019_agent_message_content.py b/alembic/versions/0019_agent_message_content.py new file mode 100644 index 0000000..17c52a3 --- /dev/null +++ b/alembic/versions/0019_agent_message_content.py @@ -0,0 +1,85 @@ +"""Add conversation-content columns to agent_messages (DB becomes primary store) + +Revision ID: 0019 +Revises: 0018 +Create Date: 2026-07-20 00:00:00.000000 + +Makes the local DB the primary store for agent conversations: agent_messages now +carries the message body and sender metadata (previously only in Slack + the +in-memory MessageLog), plus nullable Slack-mirror mapping columns. See +specs/local-db-conversations.md. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0019" +down_revision: Union[str, None] = "0018" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Content columns — DB is now the durable conversation store. + op.add_column( + "agent_messages", + sa.Column("content", sa.Text(), nullable=False, server_default=""), + ) + op.add_column( + "agent_messages", + sa.Column("sender_name", sa.String(100), nullable=False, server_default=""), + ) + op.add_column( + "agent_messages", + sa.Column("is_bot", sa.Boolean(), nullable=False, server_default=sa.text("true")), + ) + op.add_column( + "agent_messages", + sa.Column("posted_at", sa.Float(), nullable=False, server_default="0"), + ) + # Slack-mirror mapping (NULL when Slack is off / message is DB-origin). + op.add_column("agent_messages", sa.Column("slack_ts", sa.String(50), nullable=True)) + op.add_column("agent_messages", sa.Column("slack_channel_id", sa.String(100), nullable=True)) + op.add_column("agent_messages", sa.Column("slack_thread_ts", sa.String(50), nullable=True)) + + # agent_id becomes the sender_agent_id: NULL for human/PI messages. + op.alter_column("agent_messages", "agent_id", existing_type=sa.String(50), nullable=True) + + # Idempotency + rebuild/mirror indexes. + op.create_unique_constraint( + "uq_agent_messages_run_ts", "agent_messages", ["simulation_run_id", "message_ts"] + ) + op.create_index( + "ix_agent_messages_run_posted", + "agent_messages", + ["simulation_run_id", "posted_at"], + ) + op.create_index( + "ix_agent_messages_run_channel_posted", + "agent_messages", + ["simulation_run_id", "channel_name", "posted_at"], + ) + op.create_index( + "ix_agent_messages_run_slack_ts", + "agent_messages", + ["simulation_run_id", "slack_ts"], + postgresql_where=sa.text("slack_ts IS NOT NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_agent_messages_run_slack_ts", table_name="agent_messages") + op.drop_index("ix_agent_messages_run_channel_posted", table_name="agent_messages") + op.drop_index("ix_agent_messages_run_posted", table_name="agent_messages") + op.drop_constraint("uq_agent_messages_run_ts", "agent_messages", type_="unique") + op.alter_column("agent_messages", "agent_id", existing_type=sa.String(50), nullable=False) + op.drop_column("agent_messages", "slack_thread_ts") + op.drop_column("agent_messages", "slack_channel_id") + op.drop_column("agent_messages", "slack_ts") + op.drop_column("agent_messages", "posted_at") + op.drop_column("agent_messages", "is_bot") + op.drop_column("agent_messages", "sender_name") + op.drop_column("agent_messages", "content") diff --git a/alembic/versions/0020_pi_dm_messages.py b/alembic/versions/0020_pi_dm_messages.py new file mode 100644 index 0000000..b2acd57 --- /dev/null +++ b/alembic/versions/0020_pi_dm_messages.py @@ -0,0 +1,66 @@ +"""Add pi_dm_messages table (durable PI<->bot direct messages) + +Revision ID: 0020 +Revises: 0019 +Create Date: 2026-07-20 00:00:00.000000 + +DMs never entered the shared message log, so they had no durable home. This +table stores them so a PI can DM their bot (standing instructions, questions) +with Slack fully off. See specs/local-db-conversations.md. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision: str = "0020" +down_revision: Union[str, None] = "0019" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "pi_dm_messages", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column( + "simulation_run_id", + postgresql.UUID(as_uuid=True), + sa.ForeignKey("simulation_runs.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("agent_id", sa.String(50), nullable=False), + sa.Column("pi_user_id", sa.String(50), nullable=False), + sa.Column( + "direction", + sa.Enum("inbound", "outbound", name="pi_dm_direction_enum"), + nullable=False, + ), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("sender_name", sa.String(100), nullable=False, server_default=""), + sa.Column("ts", sa.String(50), nullable=False), + sa.Column("slack_ts", sa.String(50), nullable=True), + sa.Column("posted_at", sa.Float(), nullable=False, server_default="0"), + sa.Column( + "created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False, + ), + ) + op.create_index( + "ix_pi_dm_run_agent_posted", "pi_dm_messages", + ["simulation_run_id", "agent_id", "posted_at"], + ) + op.create_index( + "ix_pi_dm_run_direction_posted", "pi_dm_messages", + ["simulation_run_id", "direction", "posted_at"], + ) + + +def downgrade() -> None: + op.drop_index("ix_pi_dm_run_direction_posted", table_name="pi_dm_messages") + op.drop_index("ix_pi_dm_run_agent_posted", table_name="pi_dm_messages") + op.drop_table("pi_dm_messages") + sa.Enum(name="pi_dm_direction_enum").drop(op.get_bind(), checkfirst=True) diff --git a/alembic/versions/0021_inbox_cursor_created_at_indexes.py b/alembic/versions/0021_inbox_cursor_created_at_indexes.py new file mode 100644 index 0000000..c2cc1ae --- /dev/null +++ b/alembic/versions/0021_inbox_cursor_created_at_indexes.py @@ -0,0 +1,39 @@ +"""Index the DB inbox pollers' created_at cursor + +Revision ID: 0021 +Revises: 0020 +Create Date: 2026-07-25 00:00:00.000000 + +Both DB inbox pollers used to page over ``posted_at``, which is derived from the +*writing process's* clock (float of its minted ts). That made inbound PI delivery +depend on every writer's clock agreeing with the engine's to within the lookback +window — fine on one host, silently lossy across hosts. They now page over +``created_at`` (``server_default=now()``, i.e. the single Postgres server's +clock), so these indexes back the new access path. See +.notes/db-conversations-residual-2026-07-24.md (R3). +""" + +from typing import Sequence, Union + +from alembic import op + +revision: str = "0021" +down_revision: Union[str, None] = "0020" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_index( + "ix_agent_messages_run_created", "agent_messages", + ["simulation_run_id", "created_at"], + ) + op.create_index( + "ix_pi_dm_run_direction_created", "pi_dm_messages", + ["simulation_run_id", "direction", "created_at"], + ) + + +def downgrade() -> None: + op.drop_index("ix_pi_dm_run_direction_created", table_name="pi_dm_messages") + op.drop_index("ix_agent_messages_run_created", table_name="agent_messages") diff --git a/alembic/versions/0022_add_cohorts.py b/alembic/versions/0022_add_cohorts.py new file mode 100644 index 0000000..5a594ef --- /dev/null +++ b/alembic/versions/0022_add_cohorts.py @@ -0,0 +1,149 @@ +"""Add cohorts, cohort_memberships and cohort_audit_events + +Revision ID: 0022 +Revises: 0021 +Create Date: 2026-07-30 00:00:00.000000 + +Renumbered from 0019 at merge time. The cohort branch was cut before main's +db-primary work, so its original "0019" collided with 0019_agent_message_content: +two revisions sharing an id resolve to whichever file sorts last, which silently +skips the other while stamping the DB as fully migrated. Revision ids are assigned +at merge, never at branch. See .notes/cohort-system-v2.md §4.2 / §14 and the +alembic guard in scripts/ci.sh. + +Downgrades are idempotent (if_exists) so a rollback cannot wedge on an object that +a partially-applied upgrade never created. See v2 §14.4. +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from alembic import op + +revision: str = "0022" +down_revision: Union[str, None] = "0021" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # A cohort is a named group of agents permitted to act on each other's + # activity during simulation. See .notes/cohort-system-v2.md. + op.create_table( + "cohorts", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.String(length=48), nullable=False, unique=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column( + "created_by", + UUID(as_uuid=True), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + + # agent_id is the AgentRegistry slug (no FK — agent rows may not exist at + # membership-creation time; the app validates at add time). + op.create_table( + "cohort_memberships", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "cohort_id", + UUID(as_uuid=True), + sa.ForeignKey("cohorts.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("agent_id", sa.String(length=50), nullable=False), + sa.Column( + "added_by", + UUID(as_uuid=True), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "added_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.UniqueConstraint( + "cohort_id", "agent_id", name="uq_cohort_membership_cohort_agent" + ), + ) + op.create_index( + "ix_cohort_memberships_cohort_id", "cohort_memberships", ["cohort_id"] + ) + op.create_index( + "ix_cohort_memberships_agent_id", "cohort_memberships", ["agent_id"] + ) + + # Append-only audit trail. Deliberately denormalised: a cohort delete cascades + # its memberships away and a user delete nulls the actor FK, so the trail must + # not depend on either row surviving — hence cohort_name / actor_email columns + # and NO FK on cohort_id. `topology` snapshots the full cohort->members map + # plus the active gate settings at run start and on every change, so a + # completed simulation run stays attributable to the configuration that + # produced it (v2 §13.1). + op.create_table( + "cohort_audit_events", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("cohort_id", UUID(as_uuid=True), nullable=True), + sa.Column("cohort_name", sa.String(length=48), nullable=False), + sa.Column("agent_id", sa.String(length=50), nullable=True), + sa.Column("action", sa.String(length=32), nullable=False), + sa.Column( + "actor_id", + UUID(as_uuid=True), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("actor_email", sa.String(length=255), nullable=True), + sa.Column("simulation_run_id", UUID(as_uuid=True), nullable=True), + sa.Column("topology", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + ) + op.create_index( + "ix_cohort_audit_events_cohort_id", "cohort_audit_events", ["cohort_id"] + ) + op.create_index( + "ix_cohort_audit_events_created_at", "cohort_audit_events", ["created_at"] + ) + + +def downgrade() -> None: + op.drop_index( + "ix_cohort_audit_events_created_at", + table_name="cohort_audit_events", + if_exists=True, + ) + op.drop_index( + "ix_cohort_audit_events_cohort_id", + table_name="cohort_audit_events", + if_exists=True, + ) + op.drop_table("cohort_audit_events", if_exists=True) + op.drop_index( + "ix_cohort_memberships_agent_id", + table_name="cohort_memberships", + if_exists=True, + ) + op.drop_index( + "ix_cohort_memberships_cohort_id", + table_name="cohort_memberships", + if_exists=True, + ) + op.drop_table("cohort_memberships", if_exists=True) + op.drop_table("cohorts", if_exists=True) diff --git a/alembic/versions/0023_profile_synthesis_provenance.py b/alembic/versions/0023_profile_synthesis_provenance.py new file mode 100644 index 0000000..889c15e --- /dev/null +++ b/alembic/versions/0023_profile_synthesis_provenance.py @@ -0,0 +1,62 @@ +"""Add synthesis-provenance columns to researcher_profiles + +Revision ID: 0023 +Revises: 0022 +Create Date: 2026-07-31 00:00:00.000000 + +Two defects in src/services/profile_pipeline.py were invisible because the +pipeline wrote down nothing about *how* a profile was produced: + + 1. Step 8 computed the validation result and step 9 stored on `if synthesized:` + alone, so a profile that failed _validate_profile twice was persisted as if + it had passed. `synthesis_validated` is the record of that decision. + 2. With PubMed unreachable, ORCID works never reach the synthesis prompt + (_build_synthesis_context is fed only pubs_for_synthesis, which is derived + solely from PubMed records), so the model invents a plausible profile from + ~150 characters of name/department context and zero Publication rows are + written. `evidence_pmid_count` / `evidence_pub_count` make that case + self-identifying and separate it from a genuinely publication-less + researcher (see ResearcherProfile.evidence_state). + +All three are nullable and are deliberately NOT backfilled. NULL means "unknown +— this row predates the columns". Backfilling evidence_pub_count from +count(publications) would look like a free win and would be a lie: stored +publications accumulate across runs and include records with no abstract and +non-research article types, none of which reached any prompt. Inventing +provenance is exactly the failure these columns exist to expose. + +Downgrades are idempotent (if_exists) so a rollback cannot wedge on a column a +partially-applied upgrade never created (see scripts/ci.sh and the 0022 note). +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0023" +down_revision: Union[str, None] = "0022" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "researcher_profiles", + sa.Column("synthesis_validated", sa.Boolean(), nullable=True), + ) + op.add_column( + "researcher_profiles", + sa.Column("evidence_pmid_count", sa.Integer(), nullable=True), + ) + op.add_column( + "researcher_profiles", + sa.Column("evidence_pub_count", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("researcher_profiles", "evidence_pub_count", if_exists=True) + op.drop_column("researcher_profiles", "evidence_pmid_count", if_exists=True) + op.drop_column("researcher_profiles", "synthesis_validated", if_exists=True) diff --git a/alembic/versions/0024_add_agent_role.py b/alembic/versions/0024_add_agent_role.py new file mode 100644 index 0000000..f44e3e7 --- /dev/null +++ b/alembic/versions/0024_add_agent_role.py @@ -0,0 +1,35 @@ +"""Add role column to agents (per-role agent customization) + +Revision ID: 0024 +Revises: 0023 +Create Date: 2026-08-05 00:00:00.000000 + +`role` selects per-role prompt overrides (prompts/roles/{role}/) and a per-role +tool allow-list. Default 'pi_lab' == the pre-existing all-agents-identical +behaviour, so this column is a no-op until an agent is explicitly reassigned. +See docs/specs/2026-08-05-hub-bot-customization-design.md. + +Downgrade is idempotent (if_exists) per the branch convention (0022/0023). +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +revision: str = "0024" +down_revision: Union[str, None] = "0023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "agents", + sa.Column("role", sa.String(length=20), nullable=False, server_default="pi_lab"), + ) + + +def downgrade() -> None: + op.drop_column("agents", "role", if_exists=True) diff --git a/docker-compose.yml b/docker-compose.yml index d686043..cb89974 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,11 @@ services: agent: build: . command: python -m src.agent.main + # On SIGTERM the engine finishes the current turn and flushes buffered + # messages to Postgres (the durable store). The 10s default can cut that + # short mid-LLM-call, so give it room. Baked into the container's + # StopTimeout, so a bare `docker stop agent-run` gets it too. + stop_grace_period: 30s env_file: .env volumes: - .:/app diff --git a/docs/inbound-email.md b/docs/inbound-email.md new file mode 100644 index 0000000..6bdf689 --- /dev/null +++ b/docs/inbound-email.md @@ -0,0 +1,103 @@ +# Inbound email (reply-to-review) — architecture and runbook + +PIs are emailed collaboration proposals and can answer by replying: a rating +(1–4) files a `ProposalReview`, instructions reopen the proposal for +refinement. This document covers how the pipeline works, why it was dead in +production, and how to bring it up safely. + +## Architecture + +``` +PI hits "reply" ──► DNS MX (reply.copi.science) + └─► SES inbound SMTP (us-east-2) — receipt rule + └─► S3 s3://copi-inbound-email/inbound/ + └─► worker poll_inbound_emails (every 60s, + gated on ENABLE_INBOUND_EMAIL) + └─► process_inbound_email: + SES auth verdicts → auto-reply + filter → token lookup → sender + match → LLM classify → + review / instruction / help email +``` + +Outbound review emails set `Reply-To: review+@reply.copi.science` +(token = 64-char urlsafe secret stored on the `EmailNotification` row). The +worker deletes each S3 object after processing; objects that fail processing +3 times are quarantined under `failed/` for inspection. + +## Why it was dead in production (investigated 2026-08-11) + +Every layer below the outbound send was missing. In order of the mail's path: + +1. **No MX record** on `reply.copi.science` (only an A record to the EC2 + box, which listens on no SMTP port) — PI replies bounced after their mail + server gave up retrying. +2. **No S3 bucket**: `copi-inbound-email` did not exist in account + 215751090072. +3. **No SES receipt rule** delivering the reply domain to S3 (and the reply + domain was not verified for receiving). +4. **Instance role** `copi-ec2-ses-role` has send-only SES perms and no S3 + read/delete on the inbound bucket. +5. **`ENABLE_INBOUND_EMAIL` unset** in the prod `.env`, so the worker never + polled even if 1–4 had existed. + +Meanwhile the outbound emails actively told PIs to reply (129 sent by +2026-08-06; outbound was then paused by disabling notification categories in +the DB). + +## Code changes on the email-fix branch + +- Outbound review/new-proposal/welcome emails only solicit replies (and only + set `Reply-To` to the reply domain) when `ENABLE_INBOUND_EMAIL=true` — + outbound email can be re-enabled safely before inbound is provisioned. +- The SEC-5 anti-spoofing gate trusts only the topmost (SES-stamped) + `Authentication-Results` header; a sender-forged `...pass` header no longer + overrides SES's fail verdicts. +- HTML-only replies fall back to tag-stripped HTML instead of being silently + dropped. +- Auto-submitted mail (RFC 3834, e.g. out-of-office) is ignored — no help + email is sent back, so no mail loops. +- The declared per-token rate limit (10 replies/hour) is enforced. +- A poison message is quarantined to `failed/` after 3 attempts instead of + being retried every 60 seconds forever. + +## Bringing inbound email up + +Run each step with **admin** AWS credentials (the instance role cannot do +this — see finding 4): + +```bash +# 1. See what's missing: +python scripts/setup_inbound_email.py --check + +# 2. Create bucket, bucket policy, receipt rule set/rule; prints DNS + IAM steps: +python scripts/setup_inbound_email.py --provision +``` + +Then, in this order: + +1. Add the printed DNS records at the registrar (Namecheap): + `reply.copi.science. MX 10 inbound-smtp.us-east-2.amazonaws.com.` plus the + `_amazonses` TXT verification record if the domain was newly verified. +2. Attach the printed S3 policy to `copi-ec2-ses-role`. +3. Re-run `--check` until all layers are OK. +4. Set `ENABLE_INBOUND_EMAIL=true` in the prod `.env` and recreate the + worker: + `docker compose -f docker-compose.prod.yml -f docker-compose.override.yml up -d worker` +5. End-to-end test: trigger a proposal notification to a test recipient, + reply with "3 sounds great", and watch + `docker logs -f copi-python-worker-1` for `Email review created`. + Confirm the `proposal_reviews` row and the confirmation email. + +Only after step 5 passes, re-enable the notification categories that were +turned off in the DB (`email_notification_preferences.enabled`) / user +frequencies as desired. + +## Operational notes + +- The reply flow degrades safely: with `ENABLE_INBOUND_EMAIL` unset/false the + worker skips polling AND outbound emails stop soliciting replies. +- Quarantined mail lands in `s3://copi-inbound-email/failed/` — inspect and + delete manually. +- The rate limiter and quarantine counters are in-memory; a worker restart + resets them (by design — worst case is one extra processing round). diff --git a/docs/plans/2026-08-05-conversations-cohort-scope-and-threads.md b/docs/plans/2026-08-05-conversations-cohort-scope-and-threads.md new file mode 100644 index 0000000..98470be --- /dev/null +++ b/docs/plans/2026-08-05-conversations-cohort-scope-and-threads.md @@ -0,0 +1,1332 @@ +# Cohort-Scoped Conversations Feed + Threads + Topology Payload Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop the agent conversations page showing bot traffic from outside the viewing agent's cohort, make threads readable by expanding replies on click, and fix the topology matrix save that 400s with "Too many fields". + +**Architecture:** A new `src/services/conversation_feed.py` holds two functions: `resolve_agent_gate` (the viewing agent's `allowed_sender_ids`, computed with the engine's own `compute_gates`) and `gate_clause` (a SQLAlchemy `WHERE` fragment rendering the same truth table as the engine's in-memory `_entry_allowed`). The conversations route applies that clause **in SQL before `LIMIT`**, selects thread roots rather than raw messages, and attaches gated reply counts; a new endpoint returns a rendered replies partial on click. Separately, the topology form replaces 3,360 per-cell hidden inputs with 116 per-row/per-column markers whose cross product the handler reconstructs. + +**Tech Stack:** Python 3.11 (async), FastAPI + Starlette 1.4.1, SQLAlchemy 2.0 (async), Jinja2, Tailwind-in-template, vanilla JS (no framework), pytest + pytest-asyncio, testcontainers (Postgres). + +**Spec:** `docs/specs/2026-08-05-conversations-cohort-scope-and-threads-design.md` + +## Global Constraints + +- **No schema change, no alembic revision.** `message_ts`, `thread_ts`, `phase`, `visibility` all exist and are populated. `scripts/ci.sh` asserts a single alembic head — do not add one. +- **`gate_clause` must render the same truth table as `_entry_allowed`** (`src/agent/message_log.py:48-80`). Task 2 pins this with a parity test driven from the *existing* `DECISION_TABLE` in `tests/unit/test_cohort_isolation.py:194`. Do not copy that table — import it. +- **Order of clauses in `gate_clause` must match `_entry_allowed`'s order** (gate-off → human → private → NULL-fail-closed → cohort membership) so the correspondence is reviewable line by line. +- **The gate goes into SQL before `LIMIT`, never in Python after it.** `#general` carries traffic from 56 out-of-cohort labs; post-`LIMIT` filtering leaves a spoke PI with a near-empty page. +- **Never group threads on `slack_thread_ts`.** Use `thread_ts` / `message_ts`. The two diverge when a thread started Slack-off (`src/agent/message_log.py:35-40`). +- **Preserve the three-column ordering verbatim** — `posted_at DESC, created_at DESC, id DESC` — and its comment at `src/routers/agent_page.py:761-769`. Migration 0019 gave `posted_at` a `server_default '0'`, so pre-migration rows form one tie group and a 2-column sort makes row *selection* plan-dependent. +- **`VISIBILITY_COLLAB_PRIVATE` is imported from `src/visibility.py`**, never written as a string literal. +- **`collab_private` keeps its blanket pass.** The private-channel *read* leak is explicitly out of scope (spec §2.1). Do not add a `private_channel_members` check. +- **Run tests inside the container** per `CLAUDE.md`, with an explicit scratch DB: + ```bash + docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/ -v + ``` + Create the DB first if absent: `docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T postgres createdb -U copi copi_a3`. Never point `TEST_DATABASE_URL` at `copi`. +- **The edge-facing service is `blackbird-app`, never `app`** — a service named `app` hijacks org1's nginx upstream. Never use `--remove-orphans`. +- **Full gate before any commit is considered done:** `./scripts/ci.sh` (alembic sanity → `ruff check` on the suite → pytest with a branch-coverage floor). + +## File Structure + +| File | Responsibility | +|---|---| +| `src/services/conversation_feed.py` (new) | The only place that answers "what may this PI see in the feed, and how do I ask Postgres for it". Two functions, no route or template knowledge. | +| `src/routers/agent_page.py` (modify) | Applies the gate; selects roots + gated reply counts; serves the replies partial. | +| `templates/agent/conversations.html` (modify) | Renders roots with a reply badge and an expand control. | +| `templates/agent/_thread_replies.html` (new) | The replies fragment returned by the expand endpoint. Rendered server-side; no client templating. | +| `templates/admin/cohort_topology.html` (modify) | Emits per-row/per-column markers instead of per-cell hidden inputs. | +| `src/routers/admin.py` (modify) | Reconstructs the rendered cell set as a cross product; raises `max_fields`. | +| `tests/integration/test_conversation_feed.py` (new) | Gate parity + feed scoping + thread endpoint authz. | +| `tests/integration/test_cohort_admin.py` (modify) | Topology payload round-trip and partial-form safety. | + +--- + +### Task 1: Topology form payload — 3,360 fields → 116 + +Independent of Tasks 2-6. Land it first; it unblocks the admin UI immediately. + +**Files:** +- Modify: `templates/admin/cohort_topology.html:40-88` +- Modify: `src/routers/admin.py:1544-1569` +- Test: `tests/integration/test_cohort_admin.py` + +**Interfaces:** +- Consumes: nothing from other tasks. +- Produces: nothing other tasks rely on. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/integration/test_cohort_admin.py`. The existing helper at line 50 (`Cohort(name=name, created_by=admin.id)`) is the pattern for building cohorts; match whatever local fixture names that file already uses for `client`, `db_session`, and the admin auth header. + +```python +async def test_topology_save_round_trips_with_marker_payload( + client, db_session, admin, admin_headers +): + """The new payload adds and removes exactly the ticked/unticked cells.""" + from src.models import Cohort, CohortMembership + + c1 = Cohort(name="alpha", created_by=admin.id) + c2 = Cohort(name="beta", created_by=admin.id) + db_session.add_all([c1, c2]) + await db_session.flush() + a1 = await factories.make_agent(db_session, agent_id="ta1", bot_name="Ta1Bot") + a2 = await factories.make_agent(db_session, agent_id="ta2", bot_name="Ta2Bot") + # Pre-existing membership that the save must REMOVE (unticked but rendered). + db_session.add(CohortMembership(cohort_id=c1.id, agent_id="ta2", added_by=admin.id)) + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data=[ + ("present_agent", "ta1"), ("present_agent", "ta2"), + ("present_cohort", str(c1.id)), ("present_cohort", str(c2.id)), + ("cell", f"{c1.id}:ta1"), + ], + headers=admin_headers, + ) + assert r.status_code == 302 + assert "1+added,+1+removed" in r.headers["location"], r.headers["location"] + + rows = { + (str(cid), aid) + for cid, aid in (await db_session.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + } + assert rows == {(str(c1.id), "ta1")} + + +async def test_a_form_omitting_a_column_cannot_delete_that_columns_memberships( + client, db_session, admin, admin_headers +): + """The stale-form data-loss guard survives the cross-product reconstruction.""" + from src.models import Cohort, CohortMembership + + c1 = Cohort(name="shown", created_by=admin.id) + c2 = Cohort(name="hidden", created_by=admin.id) + db_session.add_all([c1, c2]) + await db_session.flush() + await factories.make_agent(db_session, agent_id="tb1", bot_name="Tb1Bot") + db_session.add(CohortMembership(cohort_id=c2.id, agent_id="tb1", added_by=admin.id)) + await db_session.commit() + + # c2 is NOT in present_cohort, so its cell was never rendered. + r = await client.post( + "/admin/cohorts/topology", + data=[("present_agent", "tb1"), ("present_cohort", str(c1.id))], + headers=admin_headers, + ) + assert r.status_code == 302 + + survivors = { + (str(cid), aid) + for cid, aid in (await db_session.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + } + assert survivors == {(str(c2.id), "tb1")}, "a hidden column's membership was deleted" + + +async def test_a_form_omitting_a_row_cannot_delete_that_rows_memberships( + client, db_session, admin, admin_headers +): + from src.models import Cohort, CohortMembership + + c1 = Cohort(name="only", created_by=admin.id) + db_session.add(c1) + await db_session.flush() + await factories.make_agent(db_session, agent_id="tc1", bot_name="Tc1Bot") + await factories.make_agent(db_session, agent_id="tc2", bot_name="Tc2Bot") + db_session.add(CohortMembership(cohort_id=c1.id, agent_id="tc2", added_by=admin.id)) + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data=[("present_agent", "tc1"), ("present_cohort", str(c1.id))], + headers=admin_headers, + ) + assert r.status_code == 302 + + survivors = { + aid for (aid,) in (await db_session.execute( + select(CohortMembership.agent_id) + )).all() + } + assert survivors == {"tc2"}, "a hidden row's membership was deleted" + + +async def test_full_matrix_payload_stays_under_the_field_limit( + client, db_session, admin, admin_headers +): + """60x56 used to post 3,528 fields against Starlette's max_fields=1000.""" + from src.models import Cohort + + cohorts = [] + for i in range(56): + c = Cohort(name=f"c{i:03d}", created_by=admin.id) + db_session.add(c) + cohorts.append(c) + await db_session.flush() + for i in range(60): + await factories.make_agent( + db_session, agent_id=f"td{i:03d}", bot_name=f"Td{i:03d}Bot" + ) + await db_session.commit() + + payload = ( + [("present_agent", f"td{i:03d}") for i in range(60)] + + [("present_cohort", str(c.id)) for c in cohorts] + + [("cell", f"{cohorts[0].id}:td000")] + ) + assert len(payload) == 117, f"expected 116 markers + 1 cell, got {len(payload)}" + + r = await client.post("/admin/cohorts/topology", data=payload, headers=admin_headers) + assert r.status_code == 302, r.text + assert "1+added" in r.headers["location"] +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_cohort_admin.py -k topology -v +``` + +Expected: FAIL. The round-trip and field-limit tests fail because the handler reads `present`, which the payload no longer sends, so it redirects to `?error=Nothing+to+save` instead of 302-ing to a `notice`. + +- [ ] **Step 3: Change the template** + +In `templates/admin/cohort_topology.html`, immediately after the `
` line (currently line 40), insert the markers: + +```html + + {# Rendered rows and columns. The save reconstructs the rendered CELL set as + the cross product of these two lists, which is exactly what the template + renders below (an unconditional nested loop). Same stale-form guarantee as + the old per-cell `present` inputs — a column or row that was not displayed + cannot be diffed away — at 116 fields instead of 3,360. #} + {% for a in agents %}{% endfor %} + {% for c in cohorts %}{% endfor %} +``` + +Then delete the per-cell hidden input and its comment (currently lines 80-82), leaving the checkbox: + +```html + + + +``` + +- [ ] **Step 4: Change the handler** + +In `src/routers/admin.py`, add the constant next to `_COHORT_NAME_RE` (line 1369): + +```python +# Starlette's request.form() defaults to max_fields=1000. The topology matrix +# posts one marker per rendered row and column plus one value per ticked cell, +# so the payload is agents + cohorts + ticked — but "ticked" alone will pass +# 1,000 on a large enough roster, so the limit is raised rather than relied on. +_TOPOLOGY_MAX_FIELDS = 50_000 +``` + +Replace `admin.py:1559-1561` (the `form`/`ticked`/`rendered` block): + +```python + form = await request.form(max_fields=_TOPOLOGY_MAX_FIELDS) + ticked = {v for v in form.getlist("cell") if isinstance(v, str)} + present_agents = {v for v in form.getlist("present_agent") if isinstance(v, str)} + present_cohorts = {v for v in form.getlist("present_cohort") if isinstance(v, str)} + rendered = {f"{cid}:{aid}" for cid in present_cohorts for aid in present_agents} +``` + +Everything below is unchanged: the `if not rendered` guard, the `ticked - rendered` malformed check, the unknown-id skip, the per-cell diff, and the audit events. + +Update the docstring's second sentence (line 1552) to describe the new payload: + +```python + """Apply a whole-matrix edit as a diff against the cells that were rendered. + + The form posts one ``cell`` value per ticked box (``{cohort_id}:{agent_id}``), + one ``present_agent`` per rendered row and one ``present_cohort`` per rendered + column; the rendered cell set is their cross product, which is what the + template renders (an unconditional nested loop). Sending markers instead of one + hidden input per cell keeps the payload at agents+cohorts fields rather than + agents*cohorts — 60x56 posted 3,528 fields and hit Starlette's + ``max_fields=1000``, which is why the matrix could not be saved at all. + + Diffing against ``rendered`` rather than against the whole table means a stale + or partial form can never delete memberships for a cohort or agent it did not + display — the usual checkbox-matrix data-loss bug. Unknown cohort/agent ids are + ignored, never written. Every add and remove is audited individually. + """ +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_cohort_admin.py -v +``` + +Expected: PASS, including the pre-existing topology tests in that file. + +- [ ] **Step 6: Verify by hand against the real matrix** + +Load `/admin/cohorts/topology`, toggle one checkbox, save. Expected: a 302 to `?notice=1+added,+0+removed` (or `0+added,+1+removed`), not a 400 `detail`. + +- [ ] **Step 7: Commit** + +```bash +git add templates/admin/cohort_topology.html src/routers/admin.py tests/integration/test_cohort_admin.py +git commit -m "fix(admin): topology matrix payload 3,360 fields -> 116 + +60x56 cells emitted one hidden 'present' input each, so a save posted ~3,528 +form fields against Starlette's max_fields=1000 default and 400'd with 'Too +many fields'. Post one marker per rendered row and column instead and +reconstruct the rendered cell set as their cross product — identical +stale-form guarantee, since the template renders a full matrix." +``` + +--- + +### Task 2: `gate_clause` + parity with the engine + +**Files:** +- Create: `src/services/conversation_feed.py` +- Test: `tests/integration/test_conversation_feed.py` + +**Interfaces:** +- Consumes: `_entry_allowed` and `DECISION_TABLE` (test-only, for parity). +- Produces: `gate_clause(gate: set[str] | None) -> ColumnElement[bool]` — a SQLAlchemy boolean expression over `AgentMessage`, safe to drop into any `.where()`. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/integration/test_conversation_feed.py +"""The conversations feed's visibility gate, and its parity with the engine. + +The page must show exactly what the viewing agent's bot is allowed to act on. +The engine decides that in memory (``_entry_allowed``); the page decides it in +SQL (``gate_clause``). Two implementations of one rule is a drift hazard, so the +parity test below drives BOTH from the engine's own ``DECISION_TABLE``. +""" + +import pytest +from sqlalchemy import select + +from src.agent.message_log import _entry_allowed +from src.models import AgentMessage +from src.services.conversation_feed import gate_clause +from tests import factories +from tests.unit.test_cohort_isolation import DECISION_TABLE, _post + +pytestmark = pytest.mark.integration + + +@pytest.mark.parametrize( + "name,kwargs,gate,expected", DECISION_TABLE, ids=[r[0] for r in DECISION_TABLE] +) +async def test_gate_clause_matches_entry_allowed( + db_session, name, kwargs, gate, expected +): + """Every row of the engine's §5.1 table, decided by SQL instead of Python.""" + run = await factories.make_simulation_run(db_session) + row_kwargs = dict(agent_id="x", is_bot=True, visibility="public") + row_kwargs.update( + {k: v for k, v in kwargs.items() if k in ("agent_id", "is_bot", "visibility")} + ) + msg = await factories.make_agent_message( + db_session, run=run, message_ts="1.0001", content="body", **row_kwargs + ) + await db_session.flush() + + found = (await db_session.execute( + select(AgentMessage.id).where( + AgentMessage.simulation_run_id == run.id, + gate_clause(gate), + ) + )).scalars().all() + sql_visible = msg.id in found + + entry_kwargs = dict(ts="1", channel="c", agent_id="x", name="X", content="") + entry_kwargs.update(kwargs) + python_visible = _entry_allowed(_post(**entry_kwargs), gate) + + assert sql_visible == expected, f"SQL disagreed with the table on: {name}" + assert sql_visible == python_visible, ( + f"gate_clause and _entry_allowed disagree on: {name}" + ) +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -v +``` + +Expected: FAIL at collection — `ModuleNotFoundError: No module named 'src.services.conversation_feed'`. + +- [ ] **Step 3: Write the implementation** + +```python +# src/services/conversation_feed.py +"""What a PI may see in their agent's conversations feed. + +The simulation engine gates what each agent may *act on* (``_entry_allowed`` in +``src/agent/message_log.py``); this module gates what that agent's PI may *read* +on the web page. They are the same rule, and they must never disagree — the same +constraint ``src/services/cohorts.py`` was written under, and for the same reason. + +``_entry_allowed`` filters ``LogEntry`` objects already in memory. The page cannot +do that: the filter has to run in SQL, before ``LIMIT``, or ``#general`` traffic +from every other cohort consumes the window and the page comes back near-empty. +So the rule is expressed twice — once as a predicate, once as a WHERE fragment — +and ``tests/integration/test_conversation_feed.py`` asserts the two agree on +every row of the engine's own decision table. +""" + +from __future__ import annotations + +from sqlalchemy import ColumnElement, and_, false, or_, true + +from src.models import AgentMessage +from src.visibility import VISIBILITY_COLLAB_PRIVATE + + +def gate_clause(gate: set[str] | None) -> ColumnElement[bool]: + """The cohort gate as a SQL predicate over ``AgentMessage``. + + Mirrors ``_entry_allowed`` clause for clause, in the same order, so the two + can be diffed by eye: + + - ``gate is None`` — no filtering for this agent (isolation off, or policy + "open" and the agent is uncohorted); + - the author is a **human** — keyed on ``is_bot``, *not* on a NULL + ``agent_id``. ``agent_messages.agent_id`` is nullable, so a bot-authored row + with a NULL ``agent_id`` would otherwise pass through the human bypass; + - the row is in a ``collab_private`` channel — a PI explicitly paired those + agents, and an admin-level grouping must not veto an explicit human pairing; + - a bot row with a NULL ``agent_id`` cannot be attributed to a cohort, so it + fails closed; + - otherwise the author must share a cohort with the viewing agent. + + ``gate`` is an EMPTY set for an uncohorted agent under + ``cohort_default_policy="isolated"``. That is the one input where the + membership branch must be dropped entirely rather than rendered as an empty + ``IN`` — hence the ``if gate else false()``. + """ + if gate is None: + return true() + return or_( + AgentMessage.is_bot.is_(False), + AgentMessage.visibility == VISIBILITY_COLLAB_PRIVATE, + and_( + AgentMessage.agent_id.is_not(None), + AgentMessage.agent_id.in_(gate), + ) if gate else false(), + ) +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -v +``` + +Expected: PASS — 10 parametrised cases, one per `DECISION_TABLE` row. + +- [ ] **Step 5: Commit** + +```bash +git add src/services/conversation_feed.py tests/integration/test_conversation_feed.py +git commit -m "feat(feed): gate_clause — the cohort gate as a SQL predicate + +Mirrors the engine's _entry_allowed clause for clause so the web page can +filter before LIMIT. Parity is pinned against the engine's own DECISION_TABLE +rather than a copy of it." +``` + +--- + +### Task 3: `resolve_agent_gate` + +**Files:** +- Modify: `src/services/conversation_feed.py` +- Test: `tests/integration/test_conversation_feed.py` + +**Interfaces:** +- Consumes: `compute_gates` (`src/services/cohorts.py:74`). +- Produces: `async resolve_agent_gate(db: AsyncSession, agent_id: str) -> set[str] | None` — the viewing agent's `allowed_sender_ids`; `None` = gate off, `set()` = isolated. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/integration/test_conversation_feed.py`: + +```python +from src.models import Cohort, CohortMembership +from src.services.conversation_feed import resolve_agent_gate + + +async def _cohort(db, name, *agent_ids): + c = Cohort(name=name) + db.add(c) + await db.flush() + for aid in agent_ids: + db.add(CohortMembership(cohort_id=c.id, agent_id=aid)) + await db.flush() + return c + + +async def test_gate_is_the_union_of_co_members(db_session, monkeypatch): + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent(db_session, agent_id="spoke1", bot_name="Spoke1Bot") + await factories.make_agent(db_session, agent_id="spoke2", bot_name="Spoke2Bot") + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "pair1", "spoke1", "hub") + await _cohort(db_session, "pair2", "spoke2", "hub") + + assert await resolve_agent_gate(db_session, "spoke1") == {"spoke1", "hub"} + assert await resolve_agent_gate(db_session, "spoke2") == {"spoke2", "hub"} + assert await resolve_agent_gate(db_session, "hub") == {"spoke1", "spoke2", "hub"} + + +async def test_uncohorted_agent_is_isolated_under_policy_isolated( + db_session, monkeypatch +): + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent(db_session, agent_id="lonely", bot_name="LonelyBot") + await factories.make_agent(db_session, agent_id="other", bot_name="OtherBot") + await _cohort(db_session, "somepair", "other") + + assert await resolve_agent_gate(db_session, "lonely") == set() + + +async def test_gate_is_off_when_isolation_is_disabled(db_session, monkeypatch): + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", False, raising=False) + + await factories.make_agent(db_session, agent_id="anyone", bot_name="AnyoneBot") + + assert await resolve_agent_gate(db_session, "anyone") is None + + +async def test_an_inactive_viewing_agent_still_resolves(db_session, monkeypatch): + """compute_gates only keys the roster it is given, and the conversations route + admits status 'inactive'. Without adding the viewer to the roster this raised + KeyError instead of returning a gate.""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent( + db_session, agent_id="sleeper", bot_name="SleeperBot", status="inactive" + ) + await factories.make_agent(db_session, agent_id="awake", bot_name="AwakeBot") + await _cohort(db_session, "mixed", "sleeper", "awake") + + assert await resolve_agent_gate(db_session, "sleeper") == {"sleeper", "awake"} +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -k resolve -v +``` + +Expected: FAIL — `ImportError: cannot import name 'resolve_agent_gate'`. + +- [ ] **Step 3: Write the implementation** + +Add to `src/services/conversation_feed.py` (imports first): + +```python +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.config import get_settings +from src.models import AgentRegistry, Cohort, CohortMembership +from src.services.cohorts import compute_gates +``` + +```python +async def resolve_agent_gate(db: AsyncSession, agent_id: str) -> set[str] | None: + """The viewing agent's ``allowed_sender_ids``, via the engine's own computation. + + Same call the admin preview makes (``_cohort_gate_context``), with one + deliberate difference: the roster is the active agents **plus the viewing + agent**. ``/agent/{id}/conversations`` admits ``status in ("active", + "inactive")``, but ``compute_gates`` only returns keys for the roster it is + handed, so an inactive viewer would KeyError. Adding it can only *raise* + ``live_members``, which the preflight compares against zero — so it cannot + turn a refusal into a silent roster-wide isolation. + """ + settings = get_settings() + roster = { + r[0] for r in (await db.execute( + select(AgentRegistry.agent_id).where(AgentRegistry.status == "active") + )).all() + } + roster.add(agent_id) + rows = (await db.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + cohort_count = (await db.execute( + select(func.count()).select_from(Cohort) + )).scalar() or 0 + + gates, _preflight_error = compute_gates( + membership_rows=[(r[0], r[1]) for r in rows], + agent_ids=sorted(roster), + isolation_enabled=settings.cohort_isolation_enabled, + policy=settings.cohort_default_policy, + cohort_count=cohort_count, + has_db=True, + ) + return gates.get(agent_id) +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -v +``` + +Expected: PASS, all cases including Task 2's parity set. + +- [ ] **Step 5: Commit** + +```bash +git add src/services/conversation_feed.py tests/integration/test_conversation_feed.py +git commit -m "feat(feed): resolve_agent_gate via the engine's compute_gates + +Roster is active agents plus the viewing agent, because the conversations +route admits inactive agents and compute_gates only keys its given roster." +``` + +--- + +### Task 4: Apply the gate to the feed, and select roots + +**Files:** +- Modify: `src/routers/agent_page.py:747-784` +- Test: `tests/integration/test_conversation_feed.py` + +**Interfaces:** +- Consumes: `resolve_agent_gate`, `gate_clause`. +- Produces: each entry in the template's `messages` list now carries `message_ts: str | None` and `reply_count: int` alongside the existing `channel`, `sender`, `is_bot`, `content`, `thread_ts`, `posted_at`. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/integration/test_conversation_feed.py`. `_auth` is defined in `tests/integration/test_agent_page.py:52` — import it rather than re-deriving the cookie. + +```python +from tests.integration.test_agent_page import _auth + + +async def test_a_spoke_pi_does_not_see_another_spokes_bot( + client, db_session, monkeypatch +): + """The star topology: two spokes and a hub. Spoke 1's PI must not see + Spoke 2's bot, and MUST still see the hub (the positive control).""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi1 = await factories.make_user(db_session, name="Spoke One", email="s1@example.org") + await factories.make_agent( + db_session, user=pi1, agent_id="spoke1", bot_name="Spoke1Bot", pi_name="Spoke One" + ) + await factories.make_agent(db_session, agent_id="spoke2", bot_name="Spoke2Bot") + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "pair1", "spoke1", "hub") + await _cohort(db_session, "pair2", "spoke2", "hub") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + # Spoke 1's own post is what puts #general in its channel set. + await factories.make_agent_message( + db_session, agent_id="spoke1", message_ts="1.0001", + content="MINE-own-post", sender_name="Spoke1Bot", **common + ) + await factories.make_agent_message( + db_session, agent_id="hub", message_ts="1.0002", + content="HUB-visible-post", sender_name="HubBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="spoke2", message_ts="1.0003", + content="LEAK-other-spoke-post", sender_name="Spoke2Bot", **common + ) + await db_session.commit() + + page = await client.get("/agent/spoke1/conversations", headers=_auth(pi1.id)) + assert page.status_code == 200 + assert "MINE-own-post" in page.text + assert "HUB-visible-post" in page.text, "positive control: the hub must be visible" + assert "LEAK-other-spoke-post" not in page.text + assert "Spoke2Bot" not in page.text + + +async def test_a_pi_message_still_renders_under_the_gate( + client, db_session, monkeypatch +): + """is_bot=False bypasses the gate — the human bypass must survive.""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Solo PI", email="solo@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="solo", bot_name="SoloBot", pi_name="Solo PI" + ) + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="solo", message_ts="2.0001", + content="BOT-anchor", sender_name="SoloBot", **common + ) + await factories.make_agent_message( + db_session, agent_id=None, is_bot=False, message_ts="2.0002", + content="HUMAN-said-this", sender_name="Solo PI (PI)", **common + ) + await db_session.commit() + + page = await client.get("/agent/solo/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "HUMAN-said-this" in page.text + + +async def test_replies_are_not_listed_as_top_level_rows( + client, db_session, monkeypatch +): + """The feed selects ROOTS. A reply appears via its count, not as its own card.""" + from src.config import get_settings + monkeypatch.setattr( + get_settings(), "cohort_isolation_enabled", False, raising=False + ) + + pi = await factories.make_user(db_session, name="Root PI", email="root@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="rooter", bot_name="RooterBot", pi_name="Root PI" + ) + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="rooter", message_ts="3.0001", phase="new_post", + content="THE-ROOT", sender_name="RooterBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="rooter", message_ts="3.0002", thread_ts="3.0001", + phase="thread_reply", content="THE-REPLY", sender_name="RooterBot", **common + ) + await db_session.commit() + + page = await client.get("/agent/rooter/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "THE-ROOT" in page.text + assert "THE-REPLY" not in page.text, "a reply must not render as a top-level card" + + +async def test_a_delegate_sees_exactly_what_the_owner_sees( + client, db_session, monkeypatch +): + """Access is owner-or-delegate; the gate is the AGENT's, not the viewer's, so + both must get byte-identical feeds.""" + from src.config import get_settings + from src.models import AgentDelegate + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Owner", email="own@example.org") + agent = await factories.make_agent( + db_session, user=pi, agent_id="deleg", bot_name="DelegBot", pi_name="Owner" + ) + await factories.make_agent(db_session, agent_id="stranger", bot_name="StrangerBot") + await _cohort(db_session, "solo", "deleg") + + dee = await factories.make_user(db_session, name="Dee", email="dee2@example.org") + db_session.add(AgentDelegate(agent_registry_id=agent.id, user_id=dee.id)) + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="deleg", message_ts="4.0001", + content="OWN-POST", sender_name="DelegBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="stranger", message_ts="4.0002", + content="OUTSIDER-POST", sender_name="StrangerBot", **common + ) + await db_session.commit() + + owner_page = await client.get("/agent/deleg/conversations", headers=_auth(pi.id)) + dee_page = await client.get("/agent/deleg/conversations", headers=_auth(dee.id)) + assert owner_page.status_code == 200 + assert dee_page.status_code == 200 + assert "OWN-POST" in dee_page.text + assert "OUTSIDER-POST" not in owner_page.text + assert "OUTSIDER-POST" not in dee_page.text +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -k "spoke or human or top_level or delegate" -v +``` + +Expected: FAIL — `LEAK-other-spoke-post` and `OUTSIDER-POST` are present (no gate), and `THE-REPLY` renders as its own top-level card. + +- [ ] **Step 3: Rewrite the query block** + +In `src/routers/agent_page.py`, add near the top-of-function imports (the route already imports `get_latest_run_id` inline at line 715): + +```python + from src.services.conversation_feed import gate_clause, resolve_agent_gate +``` + +Replace lines 754-784 (from the `# Recent messages ...` comment through the `messages = [...]` comprehension) with: + +```python + # What this PI may read == what their bot may act on. Filtering happens in + # SQL, before LIMIT: #general carries every other cohort's traffic, so + # filtering in Python afterwards would leave the page nearly empty. + gate = await resolve_agent_gate(db, aid) + + # Thread ROOTS, newest first. `phase` is belt-and-braces alongside + # `thread_ts IS NULL`; the two agree on every row. + # + # The three-column ordering is load-bearing, not stylistic. Migration + # 0019 adds posted_at with server_default '0', so EVERY row that + # predates it shares one value. With `ORDER BY posted_at DESC LIMIT + # 50` over a tie group larger than 50, Postgres is free to return any + # 50 — measured on a 200-row tie group, the index-scan and seq-scan + # plans returned two DISJOINT pages, so half the messages were + # unreachable and which half flipped with the plan. Adding created_at + # and the primary key makes the sort total. + root_rows = await db.execute( + select(AgentMessage) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.channel_name.in_(channels), + AgentMessage.thread_ts.is_(None), + AgentMessage.phase == "new_post", + gate_clause(gate), + ) + .order_by(AgentMessage.posted_at.desc(), AgentMessage.created_at.desc(), + AgentMessage.id.desc()) + .limit(_ROOT_LIMIT) + ) + roots = list(reversed(root_rows.scalars().all())) + + # Reply counts, gated with the SAME clause so the badge can never promise + # turns the expansion will not show. + root_ts = [r.message_ts for r in roots if r.message_ts] + counts: dict[str, int] = {} + if root_ts: + count_rows = await db.execute( + select(AgentMessage.thread_ts, func.count(AgentMessage.id)) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.thread_ts.in_(root_ts), + gate_clause(gate), + ) + .group_by(AgentMessage.thread_ts) + ) + counts = {ts: n for ts, n in count_rows} + + messages = [ + { + "channel": m.channel_name, + "sender": m.sender_name or (m.agent_id or "PI"), + "is_bot": m.is_bot, + "content": m.content, + "message_ts": m.message_ts, + "thread_ts": m.thread_ts, + "reply_count": counts.get(m.message_ts, 0), + "posted_at": m.posted_at, + } + for m in roots + ] +``` + +Add the constant near the top of `src/routers/agent_page.py`, after the imports: + +```python +# Thread roots per page. The window's unit is threads, not messages: replies no +# longer consume slots, so this surfaces more distinct conversations than the +# previous flat 100-message window did. +_ROOT_LIMIT = 50 +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py tests/integration/test_agent_page.py -v +``` + +Expected: PASS. `test_agent_page.py` must stay green — in particular +`test_posting_a_message_writes_a_pi_row_into_the_named_channel` (line ~887), whose +control asserts a PI message is visible on the read view. + +No assertion in this task depends on markup that Task 5 or 6 introduces; the +reply *badge* is asserted in Task 6, once the template that renders it exists. + +- [ ] **Step 5: Commit** + +```bash +git add src/routers/agent_page.py tests/integration/test_conversation_feed.py +git commit -m "fix(feed): cohort-scope the conversations page and select thread roots + +The feed filtered on channel name only, so every PI saw every other lab's bot +traffic in #general — contradicting the deployed star topology, where the +engine already forbids those agents from interacting. Filter with the engine's +gate in SQL before LIMIT, and select roots with gated reply counts." +``` + +--- + +### Task 5: Thread expand endpoint + replies partial + +**Files:** +- Modify: `src/routers/agent_page.py` (new route, after `agent_conversations`) +- Create: `templates/agent/_thread_replies.html` +- Test: `tests/integration/test_conversation_feed.py` + +**Interfaces:** +- Consumes: `resolve_agent_gate`, `gate_clause`, `get_agent_with_access`. +- Produces: `GET /agent/{agent_id}/thread/{message_ts}` → an HTML fragment (`200`), or `404` for any unauthorised/absent/gated-out root, or `403` from `get_agent_with_access` for a non-owner non-delegate. + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/integration/test_conversation_feed.py`: + +```python +async def _threaded_world(db_session, monkeypatch): + """Spoke1 (owned) + Spoke2 (not owned), each with a root and one reply.""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi1 = await factories.make_user(db_session, name="S One", email="t1@example.org") + await factories.make_agent( + db_session, user=pi1, agent_id="spoke1", bot_name="Spoke1Bot", pi_name="S One" + ) + await factories.make_agent(db_session, agent_id="spoke2", bot_name="Spoke2Bot") + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "p1", "spoke1", "hub") + await _cohort(db_session, "p2", "spoke2", "hub") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="spoke1", message_ts="9.0001", phase="new_post", + content="MY-ROOT", sender_name="Spoke1Bot", **common + ) + await factories.make_agent_message( + db_session, agent_id="hub", message_ts="9.0002", thread_ts="9.0001", + phase="thread_reply", content="HUB-REPLY", sender_name="HubBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="spoke2", message_ts="9.0003", phase="new_post", + content="FOREIGN-ROOT", sender_name="Spoke2Bot", **common + ) + await db_session.commit() + return pi1 + + +async def test_expanding_own_thread_returns_the_gated_replies( + client, db_session, monkeypatch +): + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/9.0001", headers=_auth(pi1.id)) + assert r.status_code == 200 + assert "HUB-REPLY" in r.text + + +async def test_expanding_an_out_of_cohort_root_is_404(client, db_session, monkeypatch): + """The IDOR guard: message_ts is guessable, so the root must re-pass the gate.""" + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/9.0003", headers=_auth(pi1.id)) + assert r.status_code == 404 + assert "FOREIGN-ROOT" not in r.text + + +async def test_expanding_a_reply_ts_rather_than_a_root_is_404( + client, db_session, monkeypatch +): + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/9.0002", headers=_auth(pi1.id)) + assert r.status_code == 404 + + +async def test_expanding_an_unknown_ts_is_404(client, db_session, monkeypatch): + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/0.0000", headers=_auth(pi1.id)) + assert r.status_code == 404 + + +async def test_a_stranger_cannot_expand_someone_elses_thread( + client, db_session, monkeypatch +): + await _threaded_world(db_session, monkeypatch) + stranger = await factories.make_user( + db_session, name="Nosy", email="nosy@example.org" + ) + await db_session.commit() + r = await client.get("/agent/spoke1/thread/9.0001", headers=_auth(stranger.id)) + assert r.status_code == 403 + assert "HUB-REPLY" not in r.text +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -k "expand or stranger" -v +``` + +Expected: FAIL with 404 on every case — the route does not exist yet. (The two +tests that *expect* 404 will fail too, on the `403` and `HUB-REPLY` assertions.) + +- [ ] **Step 3: Create the partial** + +```html +{# templates/agent/_thread_replies.html + Fragment returned by GET /agent/{agent_id}/thread/{message_ts}. Rendered + server-side so the gate and the markup stay in one place; the page injects it + verbatim. #} +{% if replies %} +
+ {% for m in replies %} +
+
+ {{ m.sender }}{% if not m.is_bot %} · PI{% endif %} +
+
{{ m.content }}
+
+ {% endfor %} +
+{% else %} +

No replies you can see in this thread.

+{% endif %} +``` + +- [ ] **Step 4: Add the route** + +In `src/routers/agent_page.py`, directly after `agent_conversations` (which ends at line 797): + +```python +@router.get("/{agent_id}/thread/{message_ts}", response_class=HTMLResponse) +async def agent_thread_replies( + agent_id: str, + message_ts: str, + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Replies for one thread, as an HTML fragment for the conversations page. + + ``message_ts`` is a guessable identifier, so authorisation cannot stop at the + agent: the ROOT is re-resolved under this agent's channel set and cohort gate + before any reply is read. Anything that does not resolve is a 404 — absent, + not-a-root, another channel, and out-of-cohort are deliberately + indistinguishable to the caller. + + Replies are gated too, with the same clause that produced the count on the + page, so the badge and the expansion can never disagree. This diverges from + the engine, which classifies ``get_thread_history`` as UNGATED because it is + thread-internal; here the whole point is that out-of-cohort traffic must not + be reachable by clicking. + """ + from src.services.conversation_feed import gate_clause, resolve_agent_gate + from src.services.pi_inbox import get_latest_run_id + + agent, _is_owner = await get_agent_with_access(agent_id, db, current_user) + if agent.status not in ("active", "inactive"): + raise HTTPException(status_code=404) + aid = agent.agent_id + + run_id = await get_latest_run_id(db) + if not run_id: + raise HTTPException(status_code=404) + + ch_rows = await db.execute( + select(distinct(AgentMessage.channel_name)).where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.agent_id == aid, + ) + ) + channels = sorted({r[0] for r in ch_rows} | {"general"}) + + gate = await resolve_agent_gate(db, aid) + root = (await db.execute( + select(AgentMessage) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.message_ts == message_ts, + AgentMessage.thread_ts.is_(None), + AgentMessage.channel_name.in_(channels), + gate_clause(gate), + ) + .limit(1) + )).scalar_one_or_none() + if root is None: + raise HTTPException(status_code=404) + + reply_rows = await db.execute( + select(AgentMessage) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.thread_ts == message_ts, + gate_clause(gate), + ) + .order_by(AgentMessage.posted_at.asc(), AgentMessage.created_at.asc(), + AgentMessage.id.asc()) + ) + replies = [ + { + "sender": m.sender_name or (m.agent_id or "PI"), + "is_bot": m.is_bot, + "content": m.content, + } + for m in reply_rows.scalars().all() + ] + + return templates.TemplateResponse( + request, "agent/_thread_replies.html", {"replies": replies} + ) +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -v +``` + +Expected: PASS. Every assertion in this task is against the endpoint's own +response, so nothing here waits on Task 6's template. + +- [ ] **Step 6: Commit** + +```bash +git add src/routers/agent_page.py templates/agent/_thread_replies.html tests/integration/test_conversation_feed.py +git commit -m "feat(feed): thread expand endpoint returning a gated replies partial + +The root is re-resolved under the agent's channel set and cohort gate before +any reply is read — message_ts is guessable, so agent-level authz alone would +be an IDOR. Replies are gated with the same clause that produced the badge." +``` + +--- + +### Task 6: Render roots with a reply badge and expand-on-click + +**Files:** +- Modify: `templates/agent/conversations.html:74-90` +- Test: `tests/integration/test_conversation_feed.py` + +**Interfaces:** +- Consumes: `messages[].message_ts`, `messages[].reply_count` (Task 4); `GET /agent/{id}/thread/{ts}` (Task 5); `data-reply-row` on each rendered reply (Task 5's partial). +- Produces: nothing later tasks rely on. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/integration/test_conversation_feed.py`. `_threaded_world` is defined in Task 5. + +```python +async def test_the_badge_count_equals_the_rendered_reply_count( + client, db_session, monkeypatch +): + """The badge is computed with the same gate as the expansion, so it can never + promise turns the expansion will not show.""" + pi1 = await _threaded_world(db_session, monkeypatch) + + page = await client.get("/agent/spoke1/conversations", headers=_auth(pi1.id)) + assert page.status_code == 200 + assert "1 reply" in page.text + assert "1 replies" not in page.text, "singular/plural must agree with the count" + + r = await client.get("/agent/spoke1/thread/9.0001", headers=_auth(pi1.id)) + assert r.status_code == 200 + assert r.text.count("data-reply-row") == 1 +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py -k badge -v +``` + +Expected: FAIL on `assert "1 reply" in page.text` — the template renders no badge yet. + +- [ ] **Step 3: Replace the Recent activity block** + +Replace `templates/agent/conversations.html` lines 74-90: + +```html + +

Recent activity

+ {% if messages %} +
+ {% for m in messages %} +
+
+ {{ m.sender }}{% if not m.is_bot %} · PI{% endif %} + #{{ m.channel }} +
+
{{ m.content }}
+ {% if m.reply_count and m.message_ts %} + + + {% endif %} +
+ {% endfor %} +
+ {% else %} +

No messages yet in your agent's channels.

+ {% endif %} + + + +{% endblock %} +``` + +Note the `· thread` badge is gone from the channel line: a reply no longer renders +as a top-level card, so the marker has nothing left to mark. + +- [ ] **Step 4: Run the full feed suite** + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird exec -T \ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \ + blackbird-app python -m pytest tests/integration/test_conversation_feed.py tests/integration/test_agent_page.py -v +``` + +Expected: PASS, including `test_the_badge_count_equals_the_rendered_reply_count` +and `test_replies_are_not_listed_as_top_level_rows`. + +- [ ] **Step 5: Run the whole gate** + +```bash +./scripts/ci.sh +``` + +Expected: alembic single head, `ruff check` clean, full pytest green above the +branch-coverage floor. + +- [ ] **Step 6: Verify by hand** + +Rebuild the web tier and load a spoke PI's page: + +```bash +docker compose -f docker-compose.prod.yml -p copi-blackbird up -d --build blackbird-app +``` + +Check: another spoke's bot does not appear in `#general`; the hub does; a root +with replies shows "Show N replies" and expands in place; clicking again +collapses without a second request (Network tab shows one call per thread). + +- [ ] **Step 7: Commit** + +```bash +git add templates/agent/conversations.html tests/integration/test_conversation_feed.py +git commit -m "feat(feed): render roots with a reply badge and expand-on-click + +Replies load once per thread from the gated fragment endpoint and are cached in +the DOM. The '· thread' badge is dropped: replies no longer render as +top-level cards, so it had nothing left to mark." +``` + +--- + +## Deployment note + +Web tier only — no schema change, and nothing the `agent-run` process imports at +startup. `docker compose -f docker-compose.prod.yml -p copi-blackbird up -d --build blackbird-app` +is sufficient; the simulation does **not** need restarting for this plan. + +⚠️ **This host runs two stacks.** The container named `agent-run` belongs to +**org1 production** (`/home/ubuntu/copi-python`); this instance's is +`blackbird-agent-run`. Never `docker stop`/`rm` the unprefixed name, and never +pass `--remove-orphans`. diff --git a/docs/plans/2026-08-06-hub-budget-scheduler.md b/docs/plans/2026-08-06-hub-budget-scheduler.md new file mode 100644 index 0000000..dfac436 --- /dev/null +++ b/docs/plans/2026-08-06-hub-budget-scheduler.md @@ -0,0 +1,1428 @@ +# Load-Proportional Budget and Scheduling Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stop the star-topology hub bot from being permanently benched by a uniform per-agent LLM cap, by deriving both its rate allowance and its scheduling weight from one shared load signal. + +**Architecture:** A single `_agent_load(agent)` method on `SimulationEngine` returns an agent's concurrent conversational obligations, clamped to `[1, active_thread_threshold]`. Two consumers read it: a sliding-window rate limiter that replaces the cumulative cap as the live throttle, and the turn scheduler's selection weights. Because both derive from one number they cannot disagree — which is the root cause being fixed. + +**Tech Stack:** Python 3.12, SQLAlchemy 2 async, Typer CLI, pytest, ruff. + +**Spec:** `docs/specs/2026-08-06-hub-budget-scheduler-design.md`. Read it before Task 1. + +## Audit trail + +Re-verified against the working tree at HEAD `7f6b304` on 2026-08-06. Nine commits +landed between this plan's first draft (`7c6768e`) and that HEAD; **none touched any +file this plan modifies** — they are confined to the PI-feed / cohort web layer +(`src/routers/agent_page.py`, `src/services/conversation_feed.py`, +`src/services/cohorts.py`, `src/models/cohort.py`, templates, and three integration +test files). Every line number and code anchor below was re-confirmed against that +tree, not carried over from an earlier read. + +## Global Constraints + +- **Task 0 is a hard prerequisite.** There is currently no Python test environment on + this host at all. Do not skip it and do not "verify" any step by inspection. +- Every task ends green on `./scripts/ci.sh`. That gate is the whole gate — there is no server-side CI. +- Branch coverage floor `COV_MIN=60`. Never lower it. +- `ruff` findings in `src/` must stay at or under `SRC_LINT_MAX=260`. Never raise it. +- `ruff` on `tests/` must be **zero** findings. New test code is held spotless. +- No database schema change and no alembic migration in this plan. If you find yourself writing one, stop — you have misread the design. +- `tests/integration/test_full_run_live.py:1111` must keep passing **with no edit**. It is the tripwire for Task 5; see that task. +- Work on branch `blackbird`. Commit after every task. +- Do not restart the live `blackbird-agent-run` container. Deployment is out of scope for this plan. + +--- + +### Task 0: Provision the test environment + +**Files:** none (everything here is gitignored — `.gitignore:77` covers `.venv-test/`). + +**Interfaces:** +- Consumes: nothing. +- Produces: a working `.venv-test/bin/python` with pytest, ruff, and the `[dev]` extras. + +**Why this task exists.** Verified on 2026-08-06: this host has **no** `.venv-test`, no +`.venv`, no `venv`, no `pip`, no `ensurepip`, and no `uv`. `python3 -m pip` fails with +"No module named pip". The `blackbird-app`, `worker`, and `grantbot` containers have no +pytest either — the `Dockerfile` runs `pip install --no-cache-dir .`, without the +`[dev]` extras. So: + +- every `pytest` command in Tasks 1-8 would fail with "No such file or directory"; +- `./scripts/ci.sh` aborts at its own guard ("ERROR: test venv python not found"); +- the container test command documented in `CLAUDE.md` also fails. + +Verified available for the fix: `pypi.org` reachable (HTTP 200), passwordless `sudo`, +Docker daemon up (needed by the integration tier's testcontainers). + +- [ ] **Step 1: Confirm the environment really is missing** + +```bash +ls .venv-test/bin/python 2>&1; python3 -m pip --version 2>&1; which uv 2>&1 +``` + +Expected: all three fail. If `.venv-test/bin/python` already exists, someone has +provisioned it since this plan was written — skip to Step 4 and verify. + +- [ ] **Step 2: Install the Python tooling** + +```bash +sudo apt-get update && sudo apt-get install -y python3-venv python3-pip +``` + +(`scripts/ci.sh`'s own error message suggests `uv` instead. Either works; apt is used +here because it does not require piping a remote installer script into a shell. If you +prefer uv: `curl -LsSf https://astral.sh/uv/install.sh | sh`, then +`uv venv .venv-test && uv pip install --python .venv-test/bin/python -e '.[dev]'`.) + +- [ ] **Step 3: Create the venv and install the dev extras** + +```bash +cd /home/ubuntu/blackbird-copi-science +python3 -m venv .venv-test +.venv-test/bin/python -m pip install --upgrade pip +.venv-test/bin/python -m pip install -e '.[dev]' +``` + +- [ ] **Step 4: Verify the toolchain** + +```bash +.venv-test/bin/python -m pytest --version +.venv-test/bin/python -m ruff --version +.venv-test/bin/python -c "import testcontainers, pytest_asyncio, factory; print('dev extras OK')" +``` + +Expected: versions print, and `dev extras OK`. + +- [ ] **Step 5: Establish the baseline — the suite must be green BEFORE any change** + +```bash +.venv-test/bin/python -m pytest tests/unit -q +``` + +Expected: all PASS. Record the count. + +This baseline is load-bearing. Tasks 4 and 6 change shared scheduler code, and without +a known-green starting point you cannot tell a regression you caused from one that was +already there. + +- [ ] **Step 6: Confirm the full gate runs end to end** + +```bash +./scripts/ci.sh +``` + +Expected: `==> CI passed.` This takes ~6 minutes and starts/destroys a throwaway +Postgres on `127.0.0.1:55432`. + +If it fails **before** you have changed any source, do not proceed — fix or report the +pre-existing failure first. Nothing in Tasks 1-8 is diagnosable on top of a red baseline. + +- [ ] **Step 7: Nothing to commit** + +`.venv-test/` is gitignored. Confirm the tree is clean of it: + +```bash +git status --short | grep venv || echo "clean" +``` + +--- + +### Task 1: The shared load signal + +**Files:** +- Modify: `src/agent/simulation.py` (add method after `_agent_within_budget`, which ends at line 378) +- Test: `tests/unit/test_hub_budget_scheduler.py` (create) + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `SimulationEngine._agent_load(self, agent: Agent) -> int`. Tasks 4 and 6 both call it. + +- [ ] **Step 1: Create the test file with its shared helpers and the first failing test** + +Create `tests/unit/test_hub_budget_scheduler.py`: + +```python +"""Load-proportional budget and scheduling for star topologies. + +Implements the test plan in docs/specs/2026-08-06-hub-budget-scheduler-design.md +§8. Organised by design section so a failure names the rule it broke: + +- TestAgentLoad §4.1 the shared load signal +- TestRoleRateOverride §4.4 optional per-role allowance +- TestCallLedger §4.2 record_api_call maintains both counters +- TestRateLimiter §4.2 sliding-window eligibility, and that it self-heals +- TestRestartRebuild §4.2 step 4b repopulates call_times from llm_call_logs +- TestScheduler §4.3 load-proportional weight, reactive tiebreak +- TestProductionRegression §8 the exact run-4f1e8395 state +""" + +import types + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.state import ThreadState + + +def _settings(**kw): + base = dict( + cohort_isolation_enabled=False, + cohort_default_policy="open", + max_consecutive_reactive_turns=3, + turn_delay_seconds=0.0, + active_thread_threshold=12, + llm_rate_window_seconds=600, + llm_calls_per_load_per_window=8, + ) + base.update(kw) + return types.SimpleNamespace(**base) + + +def _patch(monkeypatch, **kw): + monkeypatch.setattr("src.agent.simulation.get_settings", lambda: _settings(**kw)) + + +def _engine(agent_ids, budget_cap=0): + agents = [ + Agent(agent_id=a, bot_name=f"{a.capitalize()}Bot", pi_name=f"PI {a}") + for a in agent_ids + ] + return SimulationEngine(agents=agents, slack_clients={}, budget_cap=budget_cap) + + +def _add_threads(agent, n, *, status="active", pending=False, prefix="t"): + for i in range(n): + tid = f"{prefix}{i}" + agent.state.active_threads[tid] = ThreadState( + thread_id=tid, + channel="general", + other_agent_id=f"pi{i}", + status=status, + has_pending_reply=pending, + ) + + +class TestAgentLoad: + def test_idle_agent_has_load_one(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["hub"]) + assert eng._agent_load(eng.agents["hub"]) == 1 + + def test_load_counts_active_threads(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["hub"]) + _add_threads(eng.agents["hub"], 5) + assert eng._agent_load(eng.agents["hub"]) == 5 + + def test_non_active_threads_are_excluded(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["hub"]) + _add_threads(eng.agents["hub"], 3, status="active", prefix="a") + _add_threads(eng.agents["hub"], 4, status="closed", prefix="c") + _add_threads(eng.agents["hub"], 2, status="proposed", prefix="p") + assert eng._agent_load(eng.agents["hub"]) == 3 + + def test_load_is_clamped_at_active_thread_threshold(self, monkeypatch): + _patch(monkeypatch, active_thread_threshold=12) + eng = _engine(["hub"]) + _add_threads(eng.agents["hub"], 56) + assert eng._agent_load(eng.agents["hub"]) == 12 +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py -v` +Expected: 4 FAILs with `AttributeError: 'SimulationEngine' object has no attribute '_agent_load'` + +- [ ] **Step 3: Implement `_agent_load`** + +In `src/agent/simulation.py`, insert immediately after `_agent_within_budget` (which ends `return agent.api_call_count < self.budget_cap` at line 378) and before `_non_funding_thread_count`: + +```python + def _agent_load(self, agent: Agent) -> int: + """Concurrent conversational obligations for one agent. + + The shared signal behind BOTH the rate allowance (``_within_rate_limit``) + and the selection weight (``_select_agent``). Deriving both from one + number is the point: the failure this fixes was the limiter and the + scheduler holding contradictory views of what a hub deserves — the + reactive tier gave the blackbird hub a 7x boost while the cumulative cap + benched it for 161 consecutive turns, and the cap won, silently. See + docs/specs/2026-08-06-hub-budget-scheduler-design.md §1.4. + + Floors at 1 so an idle agent stays eligible. Ceilings at + ``active_thread_threshold`` so nothing can inflate its own allowance past + the thread cap it is already bound by — that clamp is what stops a + thread-opening runaway from financing itself (§4.1). + """ + live = sum( + 1 for t in agent.state.active_threads.values() if t.status == "active" + ) + return max(1, min(live, get_settings().active_thread_threshold)) +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py -v` +Expected: 4 PASS + +- [ ] **Step 5: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_hub_budget_scheduler.py +git add tests/unit/test_hub_budget_scheduler.py src/agent/simulation.py +git commit -m "feat(sched): _agent_load — the shared load signal" +``` + +--- + +### Task 2: Configuration knobs and the per-role override + +**Files:** +- Modify: `src/config.py` (after `max_consecutive_reactive_turns`, line 348) +- Modify: `src/agent/roles.py` (`RoleSpec` at line 30-34; `load_role` at line 72-101) +- Test: `tests/unit/test_roles.py` (append) +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `Settings.llm_rate_window_seconds: int = 600`, `Settings.llm_calls_per_load_per_window: int = 8`, and `RoleSpec.calls_per_load_per_window: int | None = None`. Task 4 reads all three. + +- [ ] **Step 1: Write the failing role-manifest tests** + +Append to `tests/unit/test_roles.py`: + +```python +def test_role_rate_override_is_read_when_positive(tmp_path, monkeypatch): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\ncalls_per_load_per_window = 20\n', + ) + assert load_role("scout_hub").calls_per_load_per_window == 20 + + +def test_role_rate_override_defaults_to_none(tmp_path, monkeypatch): + _write_role(tmp_path, monkeypatch, "scout_hub", 'label = "Scout Hub"\n') + assert load_role("scout_hub").calls_per_load_per_window is None + + +def test_role_rate_override_rejects_non_positive(tmp_path, monkeypatch, caplog): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\ncalls_per_load_per_window = 0\n', + ) + with caplog.at_level(logging.WARNING): + spec = load_role("scout_hub") + assert spec.calls_per_load_per_window is None + assert "calls_per_load_per_window" in caplog.text + + +def test_role_rate_override_rejects_non_int(tmp_path, monkeypatch, caplog): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\ncalls_per_load_per_window = "lots"\n', + ) + with caplog.at_level(logging.WARNING): + spec = load_role("scout_hub") + assert spec.calls_per_load_per_window is None + + +def test_missing_manifest_yields_no_rate_override(tmp_path, monkeypatch): + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + assert load_role("pi_lab").calls_per_load_per_window is None +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_roles.py -v -k rate_override` +Expected: FAIL with `TypeError: RoleSpec.__init__() got an unexpected keyword argument` or `AttributeError: 'RoleSpec' object has no attribute 'calls_per_load_per_window'` + +- [ ] **Step 3: Add the settings** + +In `src/config.py`, immediately after the `max_consecutive_reactive_turns: int = 3` line (line 348): + +```python + + # Load-proportional rate limiter. Replaces the cumulative --budget cap as the + # LIVE throttle: allowance = llm_calls_per_load_per_window * _agent_load(agent), + # measured over a sliding llm_rate_window_seconds. + # + # A rate self-heals — a throttled agent is eligible again as the window slides + # — where a cumulative cap benches permanently, and, because _rebuild_state + # restores api_call_count from llm_call_logs, benches permanently ACROSS + # RESTARTS. That is what took the blackbird hub off the air for 161 turns. + # + # Calibrated against run 4f1e8395: a spoke ran ~0.27 calls/10min and the hub + # ~2.6, so 8 leaves a spoke ~30x headroom while tripping a runaway (back-to-back + # calls) in ~25s. A hub at load 12 gets 96/window and trips in ~5min — the + # deliberate price of the 12x allowance. Lower this to tighten it. + # See docs/specs/2026-08-06-hub-budget-scheduler-design.md §4.2 / §5. + llm_rate_window_seconds: int = 600 + llm_calls_per_load_per_window: int = 8 +``` + +- [ ] **Step 4: Add the `RoleSpec` field** + +In `src/agent/roles.py`, replace the `RoleSpec` dataclass (lines 30-34): + +```python +@dataclass(frozen=True) +class RoleSpec: + name: str + label: str + tools: frozenset[str] + # Optional per-role override for Settings.llm_calls_per_load_per_window. + # None means "use the global setting". This exists to pin a specific agent; + # it is NOT the mechanism — the load signal is (design §4.4). No role sets it. + calls_per_load_per_window: int | None = None +``` + +- [ ] **Step 5: Parse the key in `load_role`** + +In `src/agent/roles.py`, replace the final `return` of `load_role` (currently `return RoleSpec(name=name, label=label, tools=tools)`) with: + +```python + rate = data.get("calls_per_load_per_window") + if rate is not None and not ( + isinstance(rate, int) and not isinstance(rate, bool) and rate > 0 + ): + logger.warning( + "[roles] %s: calls_per_load_per_window must be a positive int, " + "got %r — ignored", name, rate, + ) + rate = None + return RoleSpec( + name=name, label=label, tools=tools, calls_per_load_per_window=rate, + ) +``` + +`isinstance(rate, bool)` is excluded deliberately: `True` is an `int` in Python and would otherwise be accepted as an allowance of 1. + +- [ ] **Step 6: Run the role tests to verify they pass** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_roles.py -v` +Expected: all PASS (the pre-existing tests too — the new field is defaulted, so the two early-return paths in `load_role` need no edit) + +- [ ] **Step 7: Verify the settings load** + +Run: `.venv-test/bin/python -c "from src.config import Settings; s=Settings(); print(s.llm_rate_window_seconds, s.llm_calls_per_load_per_window)"` +Expected: `600 8` + +- [ ] **Step 8: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_roles.py +git add src/config.py src/agent/roles.py tests/unit/test_roles.py +git commit -m "feat(config): rate-limiter settings + optional per-role allowance" +``` + +--- + +### Task 3: The call ledger + +**Files:** +- Modify: `src/agent/state.py` (imports at line 3; `AgentState` at line 60-73) +- Modify: `src/agent/agent.py` (`__init__` around line 79) +- Modify: `src/agent/simulation.py` (lines 857, 903, 1172, 1947, 4670) +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `AgentState.call_times: deque[float]` and `Agent.record_api_call(self, now: float | None = None) -> None`. Task 4 reads `call_times`; Task 5 populates it. + +- [ ] **Step 1: Write the failing ledger tests** + +Append to `tests/unit/test_hub_budget_scheduler.py`: + +```python +class TestCallLedger: + def test_record_api_call_increments_both_counters(self): + a = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + a.record_api_call(now=100.0) + a.record_api_call(now=101.0) + assert a.api_call_count == 2 + assert list(a.state.call_times) == [100.0, 101.0] + + def test_record_api_call_defaults_to_wall_clock(self): + a = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + before = time.time() + a.record_api_call() + after = time.time() + assert a.api_call_count == 1 + assert before <= a.state.call_times[0] <= after + + def test_call_times_starts_empty(self): + a = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + assert len(a.state.call_times) == 0 + assert a.api_call_count == 0 +``` + +Add `import time` to the test file's imports (alphabetically before `import types`). + +- [ ] **Step 2: Run to verify they fail** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestCallLedger -v` +Expected: FAIL with `AttributeError: 'Agent' object has no attribute 'record_api_call'` + +- [ ] **Step 3: Add `call_times` to `AgentState`** + +In `src/agent/state.py`, change the import line 3 from: + +```python +from dataclasses import dataclass, field +``` + +to: + +```python +from collections import deque +from dataclasses import dataclass, field +``` + +Then in `AgentState`, immediately after `last_seen_cursor: float = 0.0` (line 68), add: + +```python + + # Sliding-window LLM call ledger, maintained by Agent.record_api_call. + # Distinct from Agent.api_call_count on purpose: api_call_count is LIFETIME + # accounting (it feeds the run summary and SimulationRun.total_api_calls), + # while call_times is the LIVE throttle and its entries age out. Only the + # latter gates eligibility, which is why throttling can no longer be + # permanent. See docs/specs/2026-08-06-hub-budget-scheduler-design.md §4.2. + call_times: deque[float] = field(default_factory=deque) +``` + +- [ ] **Step 4: Add `record_api_call` to `Agent`** + +In `src/agent/agent.py`, add this method immediately before the `# Profile properties` comment block (after `__init__` ends with `self.allowed_sender_ids: set[str] | None = None`): + +```python + def record_api_call(self, now: float | None = None) -> None: + """Record one LLM call against both the lifetime counter and the + sliding-window ledger. + + The single write point for both. Every call site must use this rather + than bumping ``api_call_count`` directly — a site that bumps only the + counter is invisible to the rate limiter, and a site that appends only to + the ledger corrupts ``SimulationRun.total_api_calls``. + """ + self.api_call_count += 1 + self.state.call_times.append(time.time() if now is None else now) +``` + +`src/agent/agent.py` does **not** currently import `time` — verified 2026-08-06, its +stdlib imports are `logging` (line 3) and `re` (line 4). Add it, and mind isort (`I` is +enabled): plain `import x` lines sort alphabetically before the `from x import y` block, +so `import time` goes **after `import re` on line 4**, before `from pathlib import Path`: + +```python +import logging +import re +import time +from pathlib import Path +from typing import Any +``` + +Verify: + +```bash +grep -n "^import time" src/agent/agent.py +.venv-test/bin/python -m ruff check src/agent/agent.py --select I,F +``` + +Expected: the grep prints line 5, and ruff prints nothing. + +- [ ] **Step 5: Run the ledger tests to verify they pass** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestCallLedger -v` +Expected: 3 PASS + +- [ ] **Step 6: Convert all five call sites** + +In `src/agent/simulation.py`, replace `agent.api_call_count += 1` with `agent.record_api_call()` at lines 857, 903, 1172, 1947, and 4670. Note line 4670 is indented one extra level (inside a `try:`), so preserve its indentation. + +Verify none were missed: + +```bash +grep -n "api_call_count += 1" src/agent/simulation.py +``` + +Expected: no output. (`simulation.py:3839` assigns `agent.api_call_count = r.count` — that is the rebuild, not an increment, and must stay.) + +- [ ] **Step 7: Run the simulation unit tests** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_simulation_logic.py tests/unit/test_cohort_isolation.py -q` +Expected: all PASS + +- [ ] **Step 8: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_hub_budget_scheduler.py +git add src/agent/state.py src/agent/agent.py src/agent/simulation.py tests/unit/test_hub_budget_scheduler.py +git commit -m "feat(sched): call ledger — record_api_call maintains both counters" +``` + +--- + +### Task 4: The sliding-window rate limiter + +**Files:** +- Modify: `src/agent/state.py` (`AgentState`, add `throttled`) +- Modify: `src/agent/simulation.py` (new import after line 25; `_turn_eligible` at line 655; stop message at lines 503-504; new methods after `_agent_load`; `_UNSET` after `SELECTION_RATIO_LOG_EVERY` at line 146; cache init after line 222) +- Modify: `tests/unit/test_cohort_isolation.py` (`_settings()` helper, ~line 126 — see Step 8; **this file breaks without that edit**) +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: `_agent_load` (Task 1), `Settings.llm_rate_window_seconds` / `llm_calls_per_load_per_window` and `RoleSpec.calls_per_load_per_window` (Task 2), `AgentState.call_times` (Task 3). +- Produces: `SimulationEngine._calls_per_load(self, agent: Agent) -> int` and `SimulationEngine._within_rate_limit(self, agent: Agent, now: float) -> bool`. Task 8 asserts against both. + +- [ ] **Step 1: Write the failing limiter tests** + +Append to `tests/unit/test_hub_budget_scheduler.py`: + +```python +class TestRateLimiter: + def test_under_allowance_is_eligible(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(7): + a.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(a, 1010.0) is True + + def test_at_allowance_is_throttled(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(8): + a.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(a, 1010.0) is False + + def test_throttle_self_heals_as_the_window_slides(self, monkeypatch): + """The regression test for the permanent bench. A throttled agent MUST + become eligible again once its calls age out — this is the single + property the cumulative cap did not have.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8, + llm_rate_window_seconds=600) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(8): + a.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(a, 1010.0) is False + # 700s later every recorded call is outside the 600s window. + assert eng._within_rate_limit(a, 1710.0) is True + assert len(a.state.call_times) == 0 + + def test_allowance_scales_with_load(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8, + active_thread_threshold=12) + eng = _engine(["hub"]) + hub = eng.agents["hub"] + _add_threads(hub, 12) + for i in range(50): + hub.record_api_call(now=1000.0 + i) + # load 12 -> allowance 96, so 50 calls is fine for a hub... + assert eng._within_rate_limit(hub, 1060.0) is True + # ...but the identical ledger throttles a load-1 spoke. + spoke = Agent(agent_id="spoke", bot_name="SpokeBot", pi_name="PI spoke") + for i in range(50): + spoke.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(spoke, 1060.0) is False + + def test_role_override_beats_the_global_setting(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + monkeypatch.setattr( + "src.agent.simulation.load_role", + lambda name: types.SimpleNamespace(calls_per_load_per_window=2), + ) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(3): + a.record_api_call(now=1000.0 + i) + assert eng._calls_per_load(a) == 2 + assert eng._within_rate_limit(a, 1010.0) is False + + def test_turn_eligible_requires_both_checks(self, monkeypatch): + """Legacy cumulative cap and the rate limiter compose with AND.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"], budget_cap=5) + a = eng.agents["spoke"] + # Rate limit fine (1 call), legacy cap blown (api_call_count 6 >= 5). + a.api_call_count = 6 + a.record_api_call(now=1000.0) + assert eng._turn_eligible(a, 1010.0) is False + + def test_turn_eligible_passes_when_both_pass(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"], budget_cap=0) + a = eng.agents["spoke"] + a.record_api_call(now=1000.0) + assert eng._turn_eligible(a, 1010.0) is True + + def test_throttle_transition_warns_once(self, monkeypatch, caplog): + _patch(monkeypatch, llm_calls_per_load_per_window=2) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + a.record_api_call(now=1000.0) + a.record_api_call(now=1001.0) + with caplog.at_level(logging.WARNING): + eng._within_rate_limit(a, 1010.0) + eng._within_rate_limit(a, 1011.0) + eng._within_rate_limit(a, 1012.0) + assert caplog.text.count("throttled") == 1 +``` + +Add `import logging` to the test file's imports (alphabetically first). + +- [ ] **Step 2: Run to verify they fail** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestRateLimiter -v` +Expected: FAIL with `AttributeError: 'SimulationEngine' object has no attribute '_within_rate_limit'` + +- [ ] **Step 3: Add the `throttled` flag to `AgentState`** + +In `src/agent/state.py`, immediately after the `call_times` field added in Task 3: + +```python + # True while the agent is rate-limited. Tracked only so the transition into + # throttling can be logged once instead of once per scheduler tick — a silent + # throttle is what turned the original incident into a 2.5-hour undetected + # outage. See design §6. + throttled: bool = False +``` + +- [ ] **Step 4: Implement the limiter** + +In `src/agent/simulation.py`, add a `src.agent.roles` import. The file has none today. + +**Placement is not free choice.** `pyproject.toml` sets `select = ["E", "F", "I", "UP", "B"]` +— `I` is isort, and it is enforced. The `src.agent.*` block is alphabetical: + +``` +25: from src.agent.prompt_safety import delimit +26: from src.agent.slack_client import SlackListingIncomplete, ThreadNotFound +27: from src.agent.state import PostRef, ProposalRef, ThreadState +28: from src.agent.tools import execute_tool, tools_for_role +``` + +`roles` sorts between `prompt_safety` and `slack_client`, so insert **after line 25**: + +```python +from src.agent.roles import load_role +``` + +Putting it after line 27 (between `state` and `tools`) is an I001 finding, which counts +against the `SRC_LINT_MAX=260` ratchet. Verify placement immediately: + +```bash +.venv-test/bin/python -m ruff check src/agent/simulation.py --select I +``` + +Expected: no output. + +Then add both methods immediately after `_agent_load` (from Task 1): + +```python + def _calls_per_load(self, agent: Agent) -> int: + """Per-unit-of-load LLM allowance for this agent's role. + + Cached by role NAME, so an agent flipping roles at runtime simply looks + up a different key and needs no invalidation. The only staleness is a + role.toml edited mid-run, which matches get_settings() already being + lru_cached — both need a container recreate (design §5). + + The cache exists because load_role() reads TOML from disk on every call + and this runs for every agent on every scheduler tick. + """ + cached = self._role_rate_cache.get(agent.role, _UNSET) + if cached is _UNSET: + cached = load_role(agent.role).calls_per_load_per_window + self._role_rate_cache[agent.role] = cached + if cached is not None: + return cached + return get_settings().llm_calls_per_load_per_window + + def _within_rate_limit(self, agent: Agent, now: float) -> bool: + """Sliding-window LLM rate check — the LIVE throttle. + + allowance = _calls_per_load(agent) * _agent_load(agent), over + llm_rate_window_seconds. Unlike the cumulative cap this replaces, it + self-heals: entries age out, so an agent throttled now is eligible later. + See design §4.2. + """ + allowance = self._calls_per_load(agent) * self._agent_load(agent) + window_start = now - get_settings().llm_rate_window_seconds + times = agent.state.call_times + while times and times[0] < window_start: + times.popleft() + ok = len(times) < allowance + if not ok and not agent.state.throttled: + logger.warning( + "[%s] throttled: %d LLM calls in the last %ds at load %d " + "(allowance %d). Eligible again as the window slides.", + agent.agent_id, len(times), + get_settings().llm_rate_window_seconds, + self._agent_load(agent), allowance, + ) + agent.state.throttled = not ok + return ok +``` + +Add the sentinel at module level, immediately after the `SELECTION_RATIO_LOG_EVERY` constant (find it with `grep -n "SELECTION_RATIO_LOG_EVERY = " src/agent/simulation.py`): + +```python +# Distinguishes "role has no cached rate yet" from "role's cached rate is None +# (no override)". A plain dict.get() default cannot tell those apart, so the +# cache would re-read role.toml from disk on every tick for every default role. +_UNSET = object() +``` + +And initialise the cache in `SimulationEngine.__init__`, immediately after `self.slack_enabled = slack_enabled` (line 222): + +```python + # role name -> calls_per_load_per_window override (or None). See _calls_per_load. + self._role_rate_cache: dict[str, int | None] = {} +``` + +- [ ] **Step 5: Compose both checks in `_turn_eligible`** + +In `src/agent/simulation.py`, replace the body of `_turn_eligible` (lines 655-669) with: + +```python + def _turn_eligible(self, agent: Agent, now: float) -> bool: + """Selection eligibility for one agent. + + - within the LEGACY cumulative cap. Inert by default (``budget_cap`` + defaults to 0, and ``_agent_within_budget`` short-circuits at <= 0); + armed only when an operator passes ``--budget``. Retained, not removed, + for back-compat — see design §6; + - within its sliding-window rate limit. This is the live throttle; + - past its per-agent cooldown. ``turn_delay_seconds`` throttles an + individual agent's tempo; enforcing it here (rather than as a global + ``asyncio.sleep`` after every productive turn) leaves the rest of the + roster free to act while one agent sits out. See v2 §10.3. + """ + if not self._agent_within_budget(agent): + return False + if not self._within_rate_limit(agent, now): + return False + delay = get_settings().turn_delay_seconds + if delay > 0 and (now - agent.state.last_selected) < delay: + return False + return True +``` + +- [ ] **Step 6: Reword the main loop's stop message** + +The loop's break message still says "over budget", which is now misleading — under a +rate limiter that state is transient, not terminal. In `src/agent/simulation.py`, +replace lines 502-505: + +```python + if not agent or not self._agent_within_budget(agent): + # All agents over budget + logger.info("All agents over budget or no agent selected. Stopping.") + break +``` + +with: + +```python + if not agent or not self._agent_within_budget(agent): + # No agent is currently eligible: every one is either rate-limited, + # cooling down, or over the legacy cumulative cap. Rate limiting is + # transient (the window slides), so this is no longer necessarily + # terminal — but the loop's contract is unchanged, so say what was + # observed rather than guessing which cause applied. + logger.info( + "No eligible agent (all throttled, cooling down, or over the " + "legacy --budget cap). Stopping." + ) + break +``` + +- [ ] **Step 7: Run the limiter tests to verify they pass** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py -v` +Expected: all PASS + +- [ ] **Step 8: Extend the OTHER suite's settings stub — REQUIRED, not optional** + +`tests/unit/test_cohort_isolation.py` stubs settings with a bare `SimpleNamespace` +carrying only four keys (its `_settings()` helper, ~line 126). The moment +`_turn_eligible` calls `_within_rate_limit` → `_agent_load`, every scheduler test in +that file raises `AttributeError: 'types.SimpleNamespace' object has no attribute +'active_thread_threshold'`. A `SimpleNamespace` has no defaults — an omitted key is an +exception, not a fallback. + +In `tests/unit/test_cohort_isolation.py`, add the three new keys to `_settings()`: + +```python +def _settings(**kw): + base = dict( + cohort_isolation_enabled=False, + cohort_default_policy=POLICY_OPEN, + max_consecutive_reactive_turns=3, + turn_delay_seconds=0.0, + # Required since _turn_eligible gained the rate limiter: _agent_load reads + # active_thread_threshold, _within_rate_limit reads the other two. Values are + # inert for this file — no test here opens a thread or records a call, so load + # is always 1 and the allowance is never approached. + active_thread_threshold=12, + llm_rate_window_seconds=600, + llm_calls_per_load_per_window=8, + ) + base.update(kw) + return types.SimpleNamespace(**base) +``` + +Audited blast radius. These are the only four test files that exercise +`_select_agent`/`_turn_eligible`, and only the first needs changing: + +| File | Settings source | Verdict | +|---|---|---| +| `tests/unit/test_cohort_isolation.py` | bare `SimpleNamespace`, 4 keys | **BREAKS — fix here** | +| `tests/integration/test_cohort_engine_live.py` | `real_settings().model_copy(...)` | safe — new fields get defaults | +| `tests/integration/test_cohort_scenarios.py` | `real_settings().model_copy(...)` | safe | +| `tests/integration/test_full_run_live.py` | `real_settings().model_copy(...)` | safe | + +(`tests/integration/test_cohort_real_llm.py` also builds a 4-key namespace but never +calls the scheduler — verified by grep — so it needs no change.) + +- [ ] **Step 9: Confirm the pre-existing suite still passes** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_cohort_isolation.py -v` +Expected: all PASS, including `test_budget_still_filters` (~line 1175). + +Note that `test_budget_still_filters` would pass **even without Step 8**, because +`_turn_eligible` checks the legacy cap first and short-circuits before reaching the +rate limiter. That is precisely why Step 8 is easy to skip and then discover late — +run the whole file, not just that test. + +- [ ] **Step 10: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_hub_budget_scheduler.py tests/unit/test_cohort_isolation.py +.venv-test/bin/python -m ruff check src --output-format=concise --quiet | wc -l +git add src/agent/state.py src/agent/simulation.py \ + tests/unit/test_hub_budget_scheduler.py tests/unit/test_cohort_isolation.py +git commit -m "feat(sched): sliding-window rate limiter replaces the cumulative cap" +``` + +The ruff count must be <= 260, and `tests/` must be zero. + +--- + +### Task 5: Restart rebuild — step 4b + +**Files:** +- Modify: `src/agent/simulation.py` (`_rebuild_state` step 4, lines 3822-3841) +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: `AgentState.call_times` (Task 3). +- Produces: nothing new. Behavioural only. + +**CRITICAL:** Step 4's existing `COUNT(*)` into `api_call_count` must be left byte-identical. You are **adding** step 4b, not modifying step 4. `tests/integration/test_full_run_live.py:1111` asserts `api_call_count` survives restart and must keep passing unedited — if it fails, you modified step 4. + +- [ ] **Step 1: Write the failing rebuild test** + +Append to `tests/unit/test_hub_budget_scheduler.py`: + +```python +class TestRestartRebuild: + def test_window_filter_selects_only_recent_calls(self, monkeypatch): + """Step 4b's cutoff arithmetic, isolated from the DB. + + The full DB round trip is covered by the integration suite; what matters + here is that the cutoff is `now - window` and that boundary rows are + included, since an off-by-one there silently re-creates the permanent + bench for anything on the edge. + """ + _patch(monkeypatch, llm_rate_window_seconds=600) + eng = _engine(["hub"]) + a = eng.agents["hub"] + now = 10_000.0 + # Simulate what step 4b loads: only rows at or after the cutoff. + cutoff = now - 600 + rows = [now - 1200, now - 700, now - 600, now - 100, now - 1] + a.state.call_times.extend(t for t in rows if t >= cutoff) + assert list(a.state.call_times) == [now - 600, now - 100, now - 1] + assert eng._within_rate_limit(a, now) is True + + def test_agent_whose_calls_all_predate_the_window_starts_unthrottled( + self, monkeypatch + ): + """The exact post-restart state that benched the hub: a large lifetime + count, but nothing inside the window.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["hub"]) + a = eng.agents["hub"] + a.api_call_count = 42 # rebuilt by step 4, lifetime + # step 4b found no rows inside the window + assert eng._within_rate_limit(a, 10_000.0) is True + assert eng._turn_eligible(a, 10_000.0) is True +``` + +- [ ] **Step 2: Run the characterisation tests — expect PASS, not FAIL** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestRestartRebuild -v` +Expected: both PASS **before** step 4b exists. + +This is deliberate and is the one place in this plan that is not red-green. `call_times` +is empty by default, so the property already holds trivially; these are +*characterisation* tests that pin it so step 4b cannot silently break it. If either +FAILS here, something in Tasks 3-4 is wrong — stop and fix that before adding step 4b. + +- [ ] **Step 3: Add step 4b** + +In `src/agent/simulation.py`, immediately after step 4's `except Exception as exc: logger.warning("Failed to rebuild api_call_count: %s", exc)` (line 3840-3841) and before the `# 5. Set last_seen_cursor per agent` comment, insert: + +```python + # 4b. Rebuild the sliding-window call ledger from the same table. + # + # Deliberately SEPARATE from step 4, which stays an all-time COUNT(*): + # api_call_count is lifetime accounting (run summary, + # SimulationRun.total_api_calls) while call_times is the live throttle. + # Folding these together is the bug — it is what made an over-budget + # agent over-budget again on every restart, forever. See design §4.2. + if self.session_factory and self.simulation_run_id: + try: + from sqlalchemy import select as sa_select + + # datetime, UTC and timedelta are already module-level imports + # (simulation.py:10) — do not re-import them here. + cutoff = datetime.now(UTC) - timedelta( + seconds=get_settings().llm_rate_window_seconds + ) + async with self.session_factory() as db: + result = await db.execute( + sa_select(LlmCallLog.agent_id, LlmCallLog.created_at) + .where( + LlmCallLog.simulation_run_id == self.simulation_run_id, + LlmCallLog.created_at >= cutoff, + ) + .order_by(LlmCallLog.created_at) + ) + for r in result: + agent = self.agents.get(r.agent_id) + if agent: + agent.state.call_times.append(r.created_at.timestamp()) + except Exception as exc: + logger.warning("Failed to rebuild call_times: %s", exc) +``` + +`.order_by(created_at)` is load-bearing: `_within_rate_limit` prunes with `popleft()` and assumes the deque is oldest-first. + +- [ ] **Step 4: Run the unit tests** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py -v` +Expected: all PASS + +- [ ] **Step 5: Verify the tripwire test is untouched and still passing** + +```bash +git diff --stat tests/integration/test_full_run_live.py +``` + +Expected: no output (file unmodified). + +Run: `.venv-test/bin/python -m pytest tests/integration/test_state_rebuild.py -q` +Expected: PASS. (This needs Docker for testcontainers.) + +- [ ] **Step 6: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_hub_budget_scheduler.py +git add src/agent/simulation.py tests/unit/test_hub_budget_scheduler.py +git commit -m "feat(sched): rebuild call_times from llm_call_logs within the window" +``` + +--- + +### Task 6: Load-proportional scheduling + +**Files:** +- Modify: `src/agent/simulation.py` (`_select_agent`, lines 671-717) +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: `_agent_load` (Task 1). +- Produces: nothing new. Behavioural only. + +- [ ] **Step 1: Write the failing scheduler tests** + +Append to `tests/unit/test_hub_budget_scheduler.py`: + +```python +class TestScheduler: + def test_proactive_weight_scales_with_load(self, monkeypatch): + """A load-12 hub against 12 load-1 spokes, all equally stale, should take + ~12/(12+12) = 50% of proactive draws. Under the old agent-fair weighting + it took 1/13 = 7.7%.""" + _patch(monkeypatch, active_thread_threshold=12) + random.seed(20260806) + ids = ["hub"] + [f"pi{i}" for i in range(12)] + eng = _engine(ids) + _add_threads(eng.agents["hub"], 12) + now = time.time() + for a in eng.agents.values(): + a.state.last_selected = now - 100.0 + + picks = [eng._select_agent().agent_id for _ in range(2000)] + share = picks.count("hub") / 2000 + assert 0.42 < share < 0.58, f"hub share {share:.3f} not load-proportional" + + def test_reactive_tiebreak_no_longer_penalises_the_busy_agent( + self, monkeypatch + ): + """The hub is selected often, so its last_selected is always recent. Under + min(last_selected) it lost every tiebreak to a long-idle spoke — it was + penalised precisely for being busy. Weighted by load, it wins.""" + _patch(monkeypatch, active_thread_threshold=12) + eng = _engine(["hub", "spoke"]) + now = time.time() + _add_threads(eng.agents["hub"], 12, pending=True) + _add_threads(eng.agents["spoke"], 1, pending=True, prefix="s") + eng.agents["hub"].state.last_selected = now - 10.0 # 10 * 12 = 120 + eng.agents["spoke"].state.last_selected = now - 60.0 # 60 * 1 = 60 + + assert eng._select_agent().agent_id == "hub" + + def test_reactive_tier_still_prefers_a_genuinely_starved_spoke( + self, monkeypatch + ): + """The load weighting must not become a blank cheque: a spoke that has + waited long enough still outranks the hub.""" + _patch(monkeypatch, active_thread_threshold=12) + eng = _engine(["hub", "spoke"]) + now = time.time() + _add_threads(eng.agents["hub"], 2, pending=True) + _add_threads(eng.agents["spoke"], 1, pending=True, prefix="s") + eng.agents["hub"].state.last_selected = now - 10.0 # 10 * 2 = 20 + eng.agents["spoke"].state.last_selected = now - 600.0 # 600 * 1 = 600 + + assert eng._select_agent().agent_id == "spoke" + + def test_throttled_hub_is_not_selected(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=1, + active_thread_threshold=12) + eng = _engine(["hub", "spoke"]) + now = time.time() + hub = eng.agents["hub"] + _add_threads(hub, 1) + hub.record_api_call(now=now) + for _ in range(50): + assert eng._select_agent().agent_id == "spoke" +``` + +Add `import random` to the test file's imports (alphabetically after `import logging`). + +- [ ] **Step 2: Run to verify they fail** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestScheduler -v` +Expected: `test_proactive_weight_scales_with_load` FAILs (share ≈ 0.077) and `test_reactive_tiebreak_no_longer_penalises_the_busy_agent` FAILs (returns `spoke`) + +- [ ] **Step 3: Weight the reactive tiebreak by load** + +In `src/agent/simulation.py`, inside `_select_agent`, replace: + +```python + return min(owed, key=lambda a: a.state.last_selected) +``` + +with: + +```python + # Weighted by load, NOT bare last_selected. The hub is selected + # often, so its last_selected is always recent — under + # min(last_selected) it lost every tiebreak to a long-idle spoke, + # i.e. it was penalised precisely for being the busiest agent. + # Still "longest wait wins", now scaled by obligation count. + # See design §1.3 / §4.3. + return max( + owed, + key=lambda a: (now - a.state.last_selected) * self._agent_load(a), + ) +``` + +- [ ] **Step 4: Weight the proactive tier by load** + +In the same method, replace: + +```python + w = max(now - a.state.last_selected, 1.0) +``` + +with: + +```python + w = max(now - a.state.last_selected, 1.0) * self._agent_load(a) +``` + +- [ ] **Step 5: Update the `_select_agent` docstring** + +Replace the docstring's item 2 (`2. **Proactive** — the original weighted-random selection:` through the `P(agent) ∝ ...` line) with: + +```python + 2. **Proactive** — staleness-weighted random, scaled by load: + P(agent) ∝ (now - last_selected) * _agent_load(agent), with a penalty + for agents that have repeatedly skipped Phase 5 + (weight /= 2^(skips-2) once skips >= 3). The load factor is what makes + a star's hub — one endpoint of every conversation — draw a share that + tracks the edges it actually sits on, instead of the 1/N a uniform + weighting gave it. See design §4.3. +``` + +- [ ] **Step 6: Run the scheduler tests to verify they pass** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py -v` +Expected: all PASS + +- [ ] **Step 7: Confirm the pre-existing scheduler suite still passes** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_cohort_isolation.py -v -k Scheduler` +Expected: all PASS. These agents have no active threads, so every load is 1 and the weighting is identity — the old assertions hold unchanged. If any fail, the load floor of 1 is not being applied. + +- [ ] **Step 8: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_hub_budget_scheduler.py +git add src/agent/simulation.py tests/unit/test_hub_budget_scheduler.py +git commit -m "feat(sched): load-proportional selection weight and reactive tiebreak" +``` + +--- + +### Task 7: Deprecate `--budget` and update the runbook + +**Files:** +- Modify: `src/agent/main.py` (line 35; `_run_simulation` around line 246) +- Modify: `CLAUDE.md` ("Running the Agent Simulation" section) +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: nothing. +- Produces: nothing. CLI + docs only. + +- [ ] **Step 1: Write the failing deprecation test** + +Append to `tests/unit/test_hub_budget_scheduler.py`: + +```python +class TestBudgetDeprecation: + def test_default_budget_is_off(self): + """The default must be 0 (off). A nonzero default is what silently armed + the legacy cap on every run.""" + import inspect + + from src.agent.main import main + + default = inspect.signature(main).parameters["budget"].default + assert default.default == 0 + + def test_help_text_marks_the_flag_deprecated(self): + import inspect + + from src.agent.main import main + + help_text = inspect.signature(main).parameters["budget"].default.help + assert "DEPRECATED" in help_text +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestBudgetDeprecation -v` +Expected: FAIL — default is 50, help text lacks "DEPRECATED" + +- [ ] **Step 3: Change the flag default and help text** + +In `src/agent/main.py`, replace line 35: + +```python + budget: int = typer.Option(50, "--budget", help="Max LLM calls per agent"), +``` + +with: + +```python + budget: int = typer.Option( + 0, "--budget", + help=( + "DEPRECATED legacy cumulative cap: max LLM calls per agent for the " + "WHOLE run. 0 (default) disables it. Superseded by the sliding-window " + "rate limiter (llm_calls_per_load_per_window). Passing a nonzero value " + "can permanently bench a hub agent — see " + "docs/specs/2026-08-06-hub-budget-scheduler-design.md §6." + ), + ), +``` + +- [ ] **Step 4: Warn loudly when the legacy cap is armed** + +In `src/agent/main.py`, immediately before the existing `logger.info("Starting simulation: ...")` call (around line 246), insert: + +```python + if budget > 0: + logger.warning( + "--budget %d is the DEPRECATED cumulative cap. It counts LLM calls " + "for the ENTIRE run, is rebuilt from llm_call_logs on restart, and " + "therefore benches an agent PERMANENTLY once crossed — this is what " + "took the blackbird hub off the air for 161 consecutive turns. The " + "sliding-window rate limiter supersedes it. Pass --budget 0 unless " + "you specifically want the legacy behaviour.", + budget, + ) +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestBudgetDeprecation -v` +Expected: 2 PASS + +- [ ] **Step 6: Verify the CLI help renders** + +Run: `.venv-test/bin/python -m src.agent.main --help` +Expected: the `--budget` entry shows the DEPRECATED text and `[default: 0]` + +- [ ] **Step 7: Update CLAUDE.md** + +In `CLAUDE.md`, under "Running the Agent Simulation", replace the four example commands so none pass a nonzero `--budget`, and add this note immediately after the code block: + +```markdown +**`--budget` is deprecated.** It is a *cumulative* cap for the whole run, it is +rebuilt from `llm_call_logs` on restart, and it therefore benches an agent +permanently once crossed — a restart does not clear it. It defaults to 0 (off) +and should stay there. Pacing and runaway protection are now handled by the +sliding-window rate limiter, whose allowance scales with each agent's live +conversational load (`llm_calls_per_load_per_window`, `llm_rate_window_seconds`). +A hub bot in a star topology will hit any uniform cumulative cap long before any +spoke does. See `docs/specs/2026-08-06-hub-budget-scheduler-design.md`. +``` + +Change the four `--budget 0` / `--budget 50` examples to drop the flag entirely, e.g.: + +```bash +# Resume an existing run: +$DC --profile agent run -d --name blackbird-agent-run agent python -m src.agent.main + +# Fresh run (wipes agent_messages/channels, keeps proposals): +$DC --profile agent run -d --name blackbird-agent-run agent python -m src.agent.main --fresh + +# With a time limit (minutes): +$DC --profile agent run -d --name blackbird-agent-run agent python -m src.agent.main --max-runtime 60 +``` + +- [ ] **Step 8: Lint and commit** + +```bash +.venv-test/bin/python -m ruff check tests/unit/test_hub_budget_scheduler.py +git add src/agent/main.py CLAUDE.md tests/unit/test_hub_budget_scheduler.py +git commit -m "feat(cli): deprecate --budget, default it off, document the replacement" +``` + +--- + +### Task 8: Production regression test and the full gate + +**Files:** +- Test: `tests/unit/test_hub_budget_scheduler.py` (append) + +**Interfaces:** +- Consumes: everything from Tasks 1-7. +- Produces: nothing. + +- [ ] **Step 1: Write the production regression test** + +Append to `tests/unit/test_hub_budget_scheduler.py`: + +```python +class TestProductionRegression: + """Reconstructs the exact state of run 4f1e8395 (2026-08-05), in which the + blackbird hub took 0 of 161 turns while 56 spokes took 3-5 each. + + Measured then: hub 42 LLM calls, next-busiest agent 9, cap 40. + """ + + def _star(self, monkeypatch, budget_cap, **kw): + _patch(monkeypatch, active_thread_threshold=12, **kw) + ids = ["blackbird"] + [f"pi{i}" for i in range(56)] + eng = _engine(ids, budget_cap=budget_cap) + eng.agents["blackbird"].api_call_count = 42 + for i in range(56): + eng.agents[f"pi{i}"].api_call_count = 8 + return eng + + def test_fixed_hub_is_selectable_after_restart(self, monkeypatch): + """Case 1 — THE FIX. New default (budget_cap=0), lifetime count 42, but + nothing inside the window because step 4b found no recent rows.""" + eng = self._star(monkeypatch, budget_cap=0) + hub = eng.agents["blackbird"] + now = time.time() + assert eng._turn_eligible(hub, now) is True + + random.seed(20260806) + picks = [eng._select_agent().agent_id for _ in range(2000)] + assert picks.count("blackbird") > 0, "hub still benched — the fix failed" + + def test_throttling_is_still_real_but_temporary(self, monkeypatch): + """Case 2 — the limiter has not been defanged. A load-1 hub that burns + its allowance inside the window IS throttled, then recovers.""" + eng = self._star(monkeypatch, budget_cap=0, + llm_calls_per_load_per_window=8, + llm_rate_window_seconds=600) + hub = eng.agents["blackbird"] + base = 10_000.0 + for i in range(8): + hub.record_api_call(now=base + i) + assert eng._turn_eligible(hub, base + 10) is False + assert eng._turn_eligible(hub, base + 700) is True + + def test_legacy_budget_flag_still_benches_the_hub(self, monkeypatch): + """Case 3 — the compat path, pinned honestly. --budget 40 was NOT made + safe; it was deprecated and defaulted off. If someone passes it, the old + behaviour is exactly what they get.""" + eng = self._star(monkeypatch, budget_cap=40) + hub = eng.agents["blackbird"] + now = time.time() + assert eng._turn_eligible(hub, now) is False + picks = [eng._select_agent().agent_id for _ in range(500)] + assert "blackbird" not in picks +``` + +- [ ] **Step 2: Run the regression test** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py::TestProductionRegression -v` +Expected: 3 PASS + +- [ ] **Step 3: Run the whole new test module** + +Run: `.venv-test/bin/python -m pytest tests/unit/test_hub_budget_scheduler.py -v` +Expected: all PASS + +- [ ] **Step 4: Run the full CI gate** + +Run: `./scripts/ci.sh` +Expected: `==> CI passed.` + +If coverage dropped below 60, add tests — do not lower `COV_MIN`. If `src/` ruff findings exceed 260, fix what you added — do not raise `SRC_LINT_MAX`. + +- [ ] **Step 5: Commit** + +```bash +git add tests/unit/test_hub_budget_scheduler.py +git commit -m "test(sched): production regression for the run-4f1e8395 hub bench" +``` + +--- + +## Deployment (NOT part of this plan) + +Applying this to the live stack is a separate, operator-gated step. It requires a +container **recreate** (not a restart) because the new settings are read through +`@lru_cache`d `get_settings()` and `env_file` is resolved at container creation. +Follow the "Before restarting" runbook in `CLAUDE.md`: save logs, `docker stop -t 30`, +`docker rm`, rebuild `blackbird-app`, then `run` the agent profile. Confirm ownership +with `docker inspect ... com.docker.compose.project` first — `copi-blackbird` is this +repo, `copi-python` is org1 and must not be touched. diff --git a/docs/plans/2026-08-10-org1-parity.md b/docs/plans/2026-08-10-org1-parity.md new file mode 100644 index 0000000..92f4e3d --- /dev/null +++ b/docs/plans/2026-08-10-org1-parity.md @@ -0,0 +1,2376 @@ +# org1 Parity Port — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bring the generic (non-Blackbird) work from `origin/blackbird` onto a branch deployable to org1/copi.science, without changing a single byte of org1's agent behaviour. + +**Architecture:** `cohort-db-conversations` is a strict ancestor of `blackbird`, so this is subtraction: replay 52 of blackbird's 119 commits verbatim (blackbird-chronological within each phase), hand-apply or trim 10 more in part, exclude 57. `copi-prod` (what org1 actually runs) is merged first so the branch can never silently revert production. No enforcement of post-type gating, no base-prompt changes, no Blackbird product. + +**Tech Stack:** git cherry-pick, Python 3 / FastAPI / SQLAlchemy async, alembic, pytest + syrupy snapshots, ruff, Docker Compose, Postgres 15. + +**Spec:** `docs/specs/2026-08-10-org1-parity-design.md` (read it before Task 1). + +## Global Constraints + +- **Branch:** `org1-parity`, currently at `17085f5`, **no upstream** (`git branch --unset-upstream` already run). Never `git push` without an explicit ` ` argument. +- **The snapshot must never move.** `git diff origin/cohort-db-conversations -- tests/characterization/__snapshots__/` must print nothing, at every task. **Never run `pytest --snapshot-update`.** +- **`src/` ruff ceiling is 260** (`SRC_LINT_MAX` in `scripts/ci.sh`). Never raise it. Measured baseline: cohort = 260, i.e. zero headroom. +- **Coverage floor is 60%** (`COV_MIN`). Never lower it. +- **Exactly one alembic head** at all times. Final head must be `0024`. +- **Never port a base prompt.** `prompts/agent-system.md`, `prompts/phase2-scan-filter.md`, `prompts/phase4-thread-reply.md`, `prompts/phase5-new-post.md` must stay byte-identical to `origin/cohort-db-conversations`. `prompts/identity.md` is new and **must contain "at Scripps Research"**. +- **Never create** `prompts/roles/scout_hub/`, `prompts/specialists/`, `src/services/patents.py`, `src/services/blackbird_rubric.py`, `src/agent/specialists.py`, `src/models/opportunity.py`, `alembic/versions/0025_*.py`, `templates/admin/assessments.html`. +- **Python interpreter for all tooling:** `.venv-test/bin/python`. Create with `uv venv .venv-test && uv pip install --python .venv-test/bin/python -e '.[dev]'` if absent. +- **Every hand-applied commit** carries a `Ported-from: (partial)` trailer and a line naming what was dropped. + +### QUICK CHECK — run at the end of every task + +```bash +# 1. Snapshot invariance. MUST print nothing. +git diff origin/cohort-db-conversations -- tests/characterization/__snapshots__/ + +# 2. Lint ratchet. MUST be <= 260. +.venv-test/bin/python -m ruff check src --output-format=concise --quiet | grep -c . + +# 3. Exactly one alembic head. +.venv-test/bin/python -m alembic heads + +# 4. Nothing left half-merged. +git status --short +``` + +### Validated lint trajectory + +Measured in a rehearsal worktree, not estimated. If a task's number differs from this table by more than 1, stop and investigate before continuing. + +| After task | `src/` findings | Note | +|---|---|---| +| baseline (`17085f5`) | 260 | ceiling is 260 | +| 1 — copi-prod merge | 260 | merge is +0 | +| 2 — Phase 1a | **255** | −5 | +| 3 — role infra picks | **257** | +2, from `4ec8ab7`'s admin role UI | +| 6 — Phase 1b | **254** | −3 | +| 7 — feed picks | **256** | +2 | +| 19 — final | **256** | comfortable | + +> Re-measured end-to-end in the 2026-08-10 audit rehearsal (full replay in a throwaway +> worktree, `./scripts/ci.sh` green at the end): 260 → 255 → 257 → 254 → 256 → 256. + +> **Why Phase 1 is split.** `3a23e73`'s `agent.py`/`main.py` import removals cannot land before Task 3: verified in rehearsal that `from typing import Any` (`agent.py:6`), `PostRef` (`agent.py:10`) and `import sys` (`main.py:13`) all survive Phase 2, and removing them first makes `ac2da9e`'s import-block patch conflict. Phase 1a (files role work never touches) goes first to buy headroom; Phase 1b goes after Phase 2. + +## File Structure + +Files this port **creates** (all new to `org1-parity`): + +| File | Responsibility | +|---|---| +| `src/agent/roles.py` | prompt-path resolution + `role.toml` manifest parsing. Dependency-free (no DB, no ORM). | +| `src/agent/post_types.py` | post-type vocabulary + role/topology filter. Dependency-free. **Inert on org1.** | +| `src/agent/thread_guidance.py` | per-role phase-4 EXPLORE/DECIDE/CONCLUDE strings. `_PI_LAB` byte-identical to the old `agent.py` literals. | +| `src/services/conversation_feed.py` | the cohort gate as a SQL predicate for PI-facing reads. | +| `alembic/versions/0024_add_agent_role.py` | one nullable-with-default column on `agents`. | +| `prompts/identity.md` | the `## Your Identity` block, extracted verbatim. **Keeps "at Scripps Research".** | +| `templates/agent/_thread_replies.html` | gated thread-expand partial. | +| `docs/specs/2026-08-06-role-topology-post-type-gating-design.md` | design doc, so `post_types.py`'s citation resolves. | + +Files this port **modifies**: `src/agent/{agent,main,simulation,state,pi_handler,tools,message_log}.py`, `src/{config,dependencies}.py`, `src/routers/{admin,agent_page,public}.py`, `src/services/{llm,cohorts}.py`, `src/models/{agent_registry,cohort}.py`, `scripts/migrate/{preflight,postflight}.py`, `scripts/migrate/run_migration.sh`, `specs/cohort-system-v2.md`, `templates/admin/{cohort_topology,agent_detail,discussions,activity_detail}.html`, `templates/agent/conversations.html`, plus tests. + +--- + +### Task 1: Merge copi-prod — the never-behind-production invariant + +**Files:** +- Modify (conflicted): `.gitignore`, `scripts/audit_pub_dois.py`, `scripts/backfill_agents.py`, `scripts/generate_sparsedata_user.py` +- Auto-merged: `src/config.py` +- Added by merge: `.dockerignore`, 9 `scripts/*.py`, `docker-compose.prod.yml` + `nginx/nginx.conf` changes + +**Interfaces:** +- Consumes: `org1-parity` @ `17085f5`. +- Produces: a branch that is a strict superset of deployed production. Every later task assumes `.dockerignore` exists and `src/config.py` has `audit_recipients`. + +**Why this is first:** deploying cohort to org1 without it would delete the tracked `.dockerignore` (so `Dockerfile`'s `COPY . .` resumes baking `.env` secrets into image layers), revert the `copi-edge` network and vhost (taking **blackbird.copi.science dark** — org1's nginx is the edge for both instances), revert `audit_recipients`, and delete 9 operator scripts. + +- [ ] **Step 1: Start the merge and confirm exactly four conflicts** + +```bash +git merge --no-ff origin/copi-prod +``` + +Expected: `Automatic merge failed; fix conflicts`. Then: + +```bash +git diff --name-only --diff-filter=U +``` + +Expected exactly: +``` +.gitignore +scripts/audit_pub_dois.py +scripts/backfill_agents.py +scripts/generate_sparsedata_user.py +``` + +If `src/config.py` appears here, stop — it auto-merged in rehearsal and a conflict means the branch is not where this plan assumes. + +- [ ] **Step 2: Resolve `.gitignore` by keeping both blocks** + +The conflict is two appended blocks. Delete the three marker lines (`<<<<<<< HEAD`, `=======`, `>>>>>>> origin/copi-prod`) and keep **all** content from both sides, HEAD's block first. Final region: + +``` +# Playwright MCP browser artifacts (console logs, page snapshots) +.playwright-mcp/ + +# Production migration dumps (scripts/migrate/run_migration.sh). Never commit these: +# they are full database copies containing message bodies and tokens. +backups/ +# Local Claude Code state +.claude/ + +# Log/backup artifacts (broadens the earlier logs/*.log rule) +logs/ + +# One-off scratch scripts and their inputs +scripts/_* + +# Editor/tool backups +*.bak.* + +# Third-party personal data (cohort seed lists, contact lists, CV/web context for +# profile synthesis, profile QA reviews, decks) lives under the ignored `data/` +# tree above — see data/cohorts/README.md. This repo is public; the roster of +# record is Postgres, not git. +``` + +- [ ] **Step 3: Resolve `scripts/audit_pub_dois.py` — take cohort's invocation AND copi-prod's redaction** + +HEAD has the current invocation but a **real ORCID**; copi-prod has a stale invocation but the redacted placeholder. Replace the whole conflict region with: + +``` + docker compose exec app python scripts/audit_pub_dois.py \\ + --orcids 0000-0000-0000-0001 --fix +``` + +The redaction is a privacy fix in a public repo. Losing it is the one unacceptable outcome of this merge. + +- [ ] **Step 4: Resolve `scripts/backfill_agents.py`** + +Replace the whole conflict region with: + +``` + docker compose cp scripts/backfill_agents.py app:/app/scripts/ + docker compose exec app python scripts/backfill_agents.py \\ + --orcids data/cohorts/newuserlist01_orcids.txt +``` + +- [ ] **Step 5: Resolve `scripts/generate_sparsedata_user.py`** + +Replace the whole conflict region with: + +``` + docker compose cp scripts/generate_sparsedata_user.py app:/app/scripts/ + docker compose exec app python scripts/generate_sparsedata_user.py \\ + --file data/cohorts/newuserlist02.tsv --force +``` + +- [ ] **Step 5b: Resolve the semantic conflict git cannot show** + +copi-prod adds a new `Settings` string field (`audit_recipients`) and cohort's +`tests/unit/test_config_secret_redaction.py` requires every string field to be +classified secret-or-not. Neither parent sees the combination, so no textual +conflict appears — but `test_every_string_field_is_classified_secret_or_not` and +`test_a_credential_in_a_url_path_is_a_known_gap` fail from this merge onward +(found at the audit rehearsal's full gate). Add at the top of +`NON_SECRET_STR_FIELDS` in `tests/unit/test_config_secret_redaction.py`: + +```python + # Comma-separated daily-audit recipient emails (copi-prod's 9ab5555) — + # addresses, not credentials. + "audit_recipients", +``` + +```bash +.venv-test/bin/python -m pytest tests/unit/test_config_secret_redaction.py -q +``` + +Expected: all pass. + +- [ ] **Step 6: Commit the merge** + +```bash +git add -A +git commit --no-edit +``` + +- [ ] **Step 7: Assert the never-behind-production invariant** + +```bash +git diff --diff-filter=A --name-only HEAD origin/copi-prod +``` + +Expected **exactly** these two lines and nothing else: +``` +templates/onboarding/add_texts.html +templates/onboarding/complete.html +``` + +Those two are the dead onboarding path cohort deleted deliberately in `d1005b1`. **Any third line is a hidden production revert — stop and investigate.** + +- [ ] **Step 8: Verify each salvaged change survived** + +```bash +grep -q 'docs/superpowers/' .gitignore && echo "ok cohort ignore rules" +grep -qx '.claude/' .gitignore && echo "ok copi-prod ignore rules" +grep -q 'docker compose exec app python scripts/audit_pub_dois.py' scripts/audit_pub_dois.py && echo "ok invocation" +grep -q '0000-0000-0000-0001' scripts/audit_pub_dois.py && ! grep -q '0000-0002-9943-7557' scripts/audit_pub_dois.py && echo "ok ORCID redacted" +grep -q 'data/cohorts/newuserlist01_orcids.txt' scripts/backfill_agents.py && echo "ok backfill args" +grep -q 'data/cohorts/newuserlist02.tsv' scripts/generate_sparsedata_user.py && echo "ok sparsedata args" +test -f .dockerignore && grep -q '^\.env' .dockerignore && echo "ok dockerignore excludes .env" +grep -q 'audit_recipients' src/config.py && echo "ok audit_recipients" +grep -q 'copi-edge' docker-compose.prod.yml && echo "ok copi-edge network" +grep -q 'blackbird' nginx/nginx.conf && echo "ok blackbird vhost" +ls scripts/vet_publications.py scripts/set_cohort_active.py >/dev/null && echo "ok maintenance scripts" +``` + +Expected: all 11 `ok` lines. + +- [ ] **Step 9: QUICK CHECK** + +Run the four QUICK CHECK commands. Expected: empty snapshot diff, ruff count **260**, one alembic head (`0023`), clean status. + +--- + +### Task 2: Buy lint headroom, part A + +**Files:** +- Modify: `src/agent/message_log.py`, `src/dependencies.py` +- Modify (tests): `tests/integration/test_proposal_review.py`, `tests/unit/test_email_templates.py`, `tests/unit/test_slack_tokens.py` + +**Interfaces:** +- Consumes: Task 1's merge. +- Produces: `src/` at **255** findings — the headroom Task 3 spends. Also makes 3 of 7 non-hermetic tests hermetic. + +**Why:** cohort sits at exactly 260/260, so Task 3 (+2) would breach the ceiling. These five files are blob-identical to `3a23e73^`, so they take cleanly, and role work never touches them. + +- [ ] **Step 1: Take the five files verbatim from `3a23e73`** + +```bash +git checkout 3a23e73 -- \ + src/agent/message_log.py \ + src/dependencies.py \ + tests/integration/test_proposal_review.py \ + tests/unit/test_email_templates.py \ + tests/unit/test_slack_tokens.py +``` + +- [ ] **Step 2: Confirm these are import-only changes to `src/`** + +```bash +git diff --cached --stat -- src/ +``` + +Expected: `src/agent/message_log.py` and `src/dependencies.py` only, each a handful of lines, all in the import block. If any hunk touches a function body, stop — you took the wrong revision. + +- [ ] **Step 3: Verify the lint count dropped to 255** + +```bash +.venv-test/bin/python -m ruff check src --output-format=concise --quiet | grep -c . +``` + +Expected: `255` + +- [ ] **Step 4: Run the affected tests** + +```bash +.venv-test/bin/python -m pytest \ + tests/unit/test_email_templates.py tests/unit/test_slack_tokens.py \ + tests/integration/test_proposal_review.py -q +``` + +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/message_log.py src/dependencies.py \ + tests/integration/test_proposal_review.py \ + tests/unit/test_email_templates.py tests/unit/test_slack_tokens.py +git commit -F - <<'MSG' +chore: clear unused imports and pin three settings-dependent tests + +src/ ruff findings 260 -> 255, buying the headroom the role and feed work +spends. cohort sat at exactly the 260 ceiling, so the next feature commit +would have failed scripts/ci.sh with no code defect. + +Three tests read SLACK_ENABLED / OUTBOUND_EMAIL_ALLOWLIST from a provisioned +.env instead of pinning what their premise depends on. They pass on a +developer checkout with a sparse .env and fail on a prod host; now they pin. + +The agent.py / main.py half of this cleanup lands separately, after the role +work — removing those imports first makes ac2da9e's import-block patch +conflict. + +Ported-from: 3a23e73 (partial) +Dropped: src/agent/agent.py, src/agent/main.py (see above), +tests/unit/test_patents.py (patents is not ported), +tests/unit/test_roles.py, tests/integration/test_agent_page.py, +tests/integration/test_cohort_admin.py (those files do not exist yet) +MSG +``` + +- [ ] **Step 6: QUICK CHECK** — ruff `255`, empty snapshot diff. + +--- + +### Task 3: Role infrastructure — 9 clean picks + +**Files:** +- Create: `src/agent/roles.py`, `prompts/identity.md`, `alembic/versions/0024_add_agent_role.py`, `tests/unit/test_roles.py`, `tests/unit/test_agent_prompts.py`, `tests/unit/test_tool_gating.py`, `tests/integration/test_role_live_flip.py` +- Modify: `src/agent/{agent,main,simulation,tools}.py`, `src/models/agent_registry.py`, `src/routers/admin.py`, `scripts/migrate/{preflight.py,run_migration.sh}`, `templates/admin/{agent_detail,cohort_topology}.html`, `tests/unit/{test_migration_checks,test_roster_sync,test_simulation_logic}.py`, `tests/integration/test_cohort_admin.py` + +**Interfaces:** +- Consumes: Task 2's headroom. +- Produces: `Agent.role: str = "pi_lab"`; `src/agent/roles.py` exporting `DEFAULT_ROLE`, `PROMPTS_DIR`, `ROLES_DIR`, `DEFAULT_TOOLS: frozenset[str]`, `RoleSpec`, `available_roles() -> list[str]`, `resolve_prompt_path(role: str, filename: str) -> Path`, `load_role(name: str) -> RoleSpec`; `src/agent/tools.py` exporting `tools_for_role(role: str) -> list[dict]` and `execute_tool(..., role: str = "pi_lab")`; alembic head `0024`. Tasks 4, 8, 9, 16, 17 all depend on these names. + +**Verified in rehearsal: all 9 apply clean, no conflicts.** + +Why it is a behavioural no-op for `pi_lab`: `resolve_prompt_path` falls through to `prompts/{filename}` when no role directory exists; `roles.DEFAULT_TOOLS` is **exactly** cohort's four `TOOL_DEFINITIONS` entries, so the allow-list grants and removes nothing; `prompts/identity.md` is byte-identical to the three literals it replaces. + +- [ ] **Step 1: Cherry-pick all nine, in this order** + +```bash +git cherry-pick a655ede 46a8391 ac2da9e 48e4d05 ffef698 da0625d 711b13b bc293d9 4ec8ab7 +``` + +Expected: nine commits, no conflict. If any stops, `git cherry-pick --abort` and re-read the spec — the order is load-bearing (`711b13b` must precede blackbird's `5367027`, which is excluded, for `tools.py` to land with only the four base tools). + +- [ ] **Step 2: Verify `prompts/identity.md` keeps the institution** + +```bash +cat prompts/identity.md +``` + +Expected exactly (no trailing newline): +``` +## Your Identity +You are **{bot_name}**, the AI agent representing the {pi_name} lab at Scripps Research. +Your agent ID is "{agent_id}". When communicating, represent your lab professionally. +``` + +If "at Scripps Research" is missing, an excluded prompt commit leaked in. Stop. + +- [ ] **Step 3: Verify no base prompt moved** + +```bash +git diff origin/cohort-db-conversations --stat -- prompts/ +``` + +Expected exactly two entries: `prompts/identity.md` (the new file) and +`prompts/daily_audit.md` (+12/−5). The `daily_audit.md` delta is **not** a leak — it +comes from Task 1's merge: copi-prod's `9ab5555` de-hardcoded the audit recipient +emails, the companion change to `src/config.py`'s `audit_recipients`. The four +agent-behaviour prompts (`agent-system`, `phase2-scan-filter`, `phase4-thread-reply`, +`phase5-new-post`) must not appear; **any of those means a base-prompt commit leaked in.** + +- [ ] **Step 4: Verify the tool allow-list is a no-op** + +```bash +.venv-test/bin/python -c " +from src.agent.roles import DEFAULT_TOOLS, load_role +from src.agent.tools import TOOL_DEFINITIONS, tools_for_role +names = {t['name'] for t in TOOL_DEFINITIONS} +assert names == set(DEFAULT_TOOLS), (names, set(DEFAULT_TOOLS)) +assert {t['name'] for t in tools_for_role('pi_lab')} == names +assert load_role('pi_lab').tools == DEFAULT_TOOLS +print('ok: pi_lab sees exactly', sorted(names)) +" +``` + +Expected: `ok: pi_lab sees exactly ['retrieve_abstract', 'retrieve_foa', 'retrieve_full_text', 'retrieve_profile']` + +- [ ] **Step 5: Verify the alembic chain** + +```bash +.venv-test/bin/python -m alembic heads +grep -E '^(revision|down_revision)' alembic/versions/0024_add_agent_role.py +``` + +Expected: `0024 (head)`; `revision: str = "0024"` and `down_revision: Union[str, None] = "0023"`. + +- [ ] **Step 6: Run the role tests** + +```bash +.venv-test/bin/python -m pytest \ + tests/unit/test_roles.py tests/unit/test_agent_prompts.py \ + tests/unit/test_tool_gating.py tests/unit/test_roster_sync.py -q +``` + +Expected: all pass. In particular `test_identity_block_is_present_and_substituted` asserts `'the Andrew Su lab at Scripps Research' in prompt` — it arrives correct from `ac2da9e` and needs **no** edit. (Blackbird's inverted version is written by the excluded `0e1ac52`.) + +- [ ] **Step 6b: Repair the one `4ec8ab7` test that assumes blackbird's prompt tree** + +`test_admin_can_set_agent_role` (`tests/integration/test_cohort_admin.py`) posts +`role=scout_hub`, which `admin_set_agent_role` validates against `available_roles()`. +On this branch `prompts/roles/` is deliberately absent, so the only assignable role is +`pi_lab`, the route correctly refuses the write, and the test fails for the wrong +reason (found at the audit rehearsal's full gate). Give the validator a real second +role the way `tests/unit/test_roles.py` does — a tmp roles dir. Change the test's +opening to: + +```python +async def test_admin_can_set_agent_role(client, db_session, admin, roster, tmp_path, monkeypatch): + # org1 ships no prompts/roles/ tree, so give the validator a real second + # role the same way tests/unit/test_roles.py does: a tmp roles dir. + from src.agent import roles as roles_mod + d = tmp_path / "roles" / "scout_hub" + d.mkdir(parents=True) + (d / "role.toml").write_text('label = "Scout Hub"\n', encoding="utf-8") + monkeypatch.setattr(roles_mod, "ROLES_DIR", tmp_path / "roles") + agent = roster["su"] +``` + +(the two auth tests that also post `scout_hub` assert 403/redirect before validation +and need no change). Then: + +```bash +.venv-test/bin/python -m pytest tests/integration/test_cohort_admin.py -q +git add tests/integration/test_cohort_admin.py +git commit -F - <<'MSG' +test(admin): give the role-set test a tmp roles dir — org1 ships no prompts/roles + +4ec8ab7's test_admin_can_set_agent_role posts role=scout_hub, which +admin_set_agent_role validates against available_roles(). On this branch +prompts/roles/ is deliberately absent, so the only assignable role is pi_lab, +the route correctly refuses the write, and the test fails for the wrong +reason. Build the second role in tmp_path exactly the way the unit tests do, +so the test exercises the mechanism instead of blackbird's shipped persona. +MSG +``` + +- [ ] **Step 7: QUICK CHECK** — ruff **257**, empty snapshot diff, one head (`0024`). + +--- + +### Task 4: Restore the phase-5 role override + +**Files:** +- Modify: `src/agent/agent.py` + +**Interfaces:** +- Consumes: `Agent._load_prompt(filename: str, default: str) -> str` from Task 3. +- Produces: `build_phase5_prompt` honouring role overrides — completing the mechanism Task 3 installed. + +**Why:** `ac2da9e` routed agent-system and identity through `_load_prompt()` (phases 2, 2-prune and 4 follow with Task 10's `2467229`/`bc40d20`), but `build_phase5_prompt` kept a hardcoded global path, so any role's phase-5 override is silently ignored. The 2-line fix lives in `6b76f27`, whose remainder is the scout_hub prompt tree. A no-op for `pi_lab` (which has no override), and `6b76f27` itself records that pi_lab's phase-5 snapshot does not move. + +- [ ] **Step 1: Find the hardcoded load** + +```bash +grep -n 'PROMPTS_DIR / "phase5-new-post.md"' src/agent/agent.py +``` + +Expected: one hit inside `build_phase5_prompt`. + +- [ ] **Step 2: Replace it with the role-aware loader** + +Change: + +```python + phase5_template = self._load_file( + PROMPTS_DIR / "phase5-new-post.md", + "Choose to reply to an interesting post or make a new top-level post.", + ) +``` + +to: + +```python + phase5_template = self._load_prompt( + "phase5-new-post.md", + "Choose to reply to an interesting post or make a new top-level post.", + ) +``` + +- [ ] **Step 3: Verify every phase now routes through `_load_prompt`** + +```bash +grep -c '_load_prompt(' src/agent/agent.py +grep -n 'PROMPTS_DIR' src/agent/agent.py +``` + +Expected **at this point**: `_load_prompt(` appears **4** times (its `def`, agent-system, identity, phase5-new-post), and `PROMPTS_DIR` still appears in its definition plus three `_load_file` calls (phase2-scan-filter, phase2-prune, phase4-thread-reply) — those three phases route through `_load_prompt` only when Task 10 picks `2467229`/`bc40d20`. Re-run this check after Task 10: **7** occurrences and no `PROMPTS_DIR` anywhere. + +- [ ] **Step 4: Confirm pi_lab behaviour is unchanged** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_agent_prompts.py tests/characterization -q +``` + +Expected: all pass, no snapshot movement. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/agent.py +git commit -F - <<'MSG' +fix(agent): phase 5 must honour role prompt overrides like every other phase + +build_phase5_prompt loaded prompts/phase5-new-post.md through a hardcoded +global path while every other phase went through _load_prompt(), so a role's +phase5 override was silently ignored. Inert for pi_lab, which has no override +— but it left the mechanism installed one phase short of complete. + +Ported-from: 6b76f27 (partial) +Dropped: prompts/roles/scout_hub/{role.toml,identity.md,agent-system.md, +phase5-new-post.md} and the tests/unit/test_roles.py scout_hub cases — the +scouting persona is Blackbird product and is not ported. +MSG +``` + +- [ ] **Step 6: QUICK CHECK** — ruff `257`, empty snapshot diff. + +--- + +### Task 5: Complete the migration tooling for 0024 + +**Files:** +- Modify: `scripts/migrate/preflight.py`, `scripts/migrate/postflight.py`, `tests/integration/test_harness_smoke.py`, `tests/unit/test_migration_checks.py`, `docs/production-migration.md` + +**Interfaces:** +- Consumes: alembic head `0024` from Task 3; `ffef698`'s `DEFAULT_TARGET = "0024"`, `SUPPORTED_START_REVISIONS`, `REVISION_ORDER`. +- Produces: preflight that plans `agents.role`, postflight that verifies it, a head pin matching reality. + +**Why:** `ffef698` (Task 3) moved the targets but not the object inventory — that lives in `517a564`, which is excluded because the rest of it is patents/0025 work. Left alone, `REVISION_ORDER` reaches `0024` while preflight's collision check silently skips `agents.role`, and `test_harness_smoke.py` still asserts `0023`. + +- [ ] **Step 1: Confirm the four gaps are real** + +```bash +grep -n 'assert v ==' tests/integration/test_harness_smoke.py +grep -c 'PlannedObject("0024"' scripts/migrate/preflight.py +grep -n 'for revision in ("0019"' tests/unit/test_migration_checks.py +grep -c '"agents", "role"' scripts/migrate/postflight.py +``` + +Expected: `assert v == "0023"`; `0`; the drift-guard tuple ending at `"0023"`; `0`. + +- [ ] **Step 2: Bump the harness head pin** + +In `tests/integration/test_harness_smoke.py`, change: + +```python + # 0023 researcher_profiles synthesis provenance + assert v == "0023" +``` + +to: + +```python + # 0023 researcher_profiles synthesis provenance, 0024 agents.role column + assert v == "0024" +``` + +- [ ] **Step 3: Add the planned object for 0024** + +In `scripts/migrate/preflight.py`, immediately after the last `PlannedObject("0023", ...)` entry and before the closing `)` of `PLANNED_OBJECTS`, add: + +```python + # 0024_add_agent_role + PlannedObject("0024", "column", "role", "agents"), +``` + +- [ ] **Step 4: Correct `ffef698`'s inherited false claim** + +`ffef698` justifies adding `"0023"` to `SUPPORTED_START_REVISIONS` as "production's current stamp". That is blackbird's stamp; org1 is at `0018`. Replace that comment line with: + +```python +#: 0023 is supported because it is where a deployment that already took the cohort +#: migration sits. org1 is at 0018 (see docs/production-migration.md); do not read +#: this tuple as a statement about any one deployment's current stamp. +``` + +- [ ] **Step 5: Widen the drift guard to 0024** + +In `tests/unit/test_migration_checks.py`, change: + +```python + for revision in ("0019", "0020", "0021", "0022", "0023"): +``` + +to: + +```python + for revision in ("0019", "0020", "0021", "0022", "0023", "0024"): +``` + +- [ ] **Step 5b: Retarget the totality test** + +`test_planned_objects_between_0018_and_0023_is_everything` asserts that the +0018→0023 range covers **all** of `PLANNED_OBJECTS`; Step 3's 0024 entry breaks it +(found in the audit rehearsal). Take the rename blackbird's `517a564` made — the +rest of that commit is patents/0025 work. In `tests/unit/test_migration_checks.py`, +change: + +```python +def test_planned_objects_between_0018_and_0023_is_everything(): + assert set(pf.planned_objects_between("0018", "0023")) == set(pf.PLANNED_OBJECTS) +``` + +to: + +```python +def test_planned_objects_between_0018_and_the_target_is_everything(): + assert set(pf.planned_objects_between("0018", pf.DEFAULT_TARGET)) == set(pf.PLANNED_OBJECTS) +``` + +- [ ] **Step 6: Make postflight verify `agents.role`** + +In `scripts/migrate/postflight.py`, append to `EXPECTED_COLUMNS` (after the last 0023 entry): + +```python + # 0024. NOT NULL with a server_default, so every existing agent reads as pi_lab + # — which is exactly the pre-0024 behaviour. + ("agents", "role", "character varying", False, "'pi_lab'::character varying"), +``` + +- [ ] **Step 7: Update the runbook's target** + +In `docs/production-migration.md`, change the title from `# Production migration to alembic 0023 (`cohort-db-conversations`)` to `# Production migration to alembic 0024 (`org1-parity`)`, and add a row to the §1 table after the `0022 -> 0023` row: + +``` +| `0023 -> 0024` | One `VARCHAR(20) NOT NULL DEFAULT 'pi_lab'` column on `agents`. Postgres 11+ fills a non-volatile default without a table rewrite, and `agents` is small, so this is seconds at any size. | +``` + +- [ ] **Step 8: Verify the tooling agrees with the migration files** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_migration_checks.py -q +.venv-test/bin/python -c " +import sys; sys.path.insert(0, 'scripts/migrate') +import preflight as pf +assert pf.DEFAULT_TARGET == '0024', pf.DEFAULT_TARGET +assert pf.REVISION_ORDER[-1] == '0024' +planned = pf.planned_objects_between('0018', '0024') +assert any(o.revision == '0024' and o.name == 'role' for o in planned), planned +print('ok: preflight plans', len(planned), 'objects for 0018 -> 0024') +" +``` + +Expected: tests pass; the `ok:` line prints. + +- [ ] **Step 9: Verify postflight still derives the same table set** + +```bash +.venv-test/bin/python -c " +import sys; sys.path.insert(0, 'scripts/migrate') +import postflight as po +assert po.CHAIN_CREATED_TABLES == {'pi_dm_messages','cohorts','cohort_memberships','cohort_audit_events'}, po.CHAIN_CREATED_TABLES +print('ok: 0024 adds no table, so CHAIN_CREATED_TABLES is unchanged') +" +``` + +Expected: the `ok:` line. This is why postflight needs no `VERIFIED_REVISIONS` guard here. + +- [ ] **Step 10: Lint the migration tooling (it is in `ci.sh`'s zero-findings set)** + +```bash +.venv-test/bin/python -m ruff check scripts/migrate +``` + +Expected: no output. + +- [ ] **Step 11: Commit** + +```bash +git add scripts/migrate/preflight.py scripts/migrate/postflight.py \ + tests/integration/test_harness_smoke.py tests/unit/test_migration_checks.py \ + docs/production-migration.md +git commit -F - <<'MSG' +fix(migrate): plan and verify 0024's column, and pin the head to it + +ffef698 moved DEFAULT_TARGET/SUPPORTED_START_REVISIONS/REVISION_ORDER to 0024 +but not the object inventory, which lives in 517a564 (excluded — the rest of +that commit is patents and 0025 work). Left alone, REVISION_ORDER reaches 0024 +while preflight's collision check silently skips agents.role, and +test_harness_smoke still asserted 0023. + +- preflight: PlannedObject("0024", "column", "role", "agents"), so a + pre-existing agents.role is detected rather than discovered mid-migration. +- postflight: EXPECTED_COLUMNS gains agents.role, so 0024 is VERIFIED after + the window rather than assumed. Blackbird documented this gap + (VERIFIED_REVISIONS) instead of closing it; closing it is cheap here because + 0024 creates no table, so CHAIN_CREATED_TABLES is untouched. +- test_migration_checks: the drift guard now re-derives 0024 from the + migration file too, and the totality test compares against DEFAULT_TARGET + instead of a hardcoded "0023" (517a564's rename), so it stops breaking every + time the chain grows. +- Reworded ffef698's comment, which called 0023 "production's current stamp". + That is blackbird's stamp. org1 is at 0018. + +Ported-from: 9714f26 (partial) +Dropped: tests/live_api/test_patents_live.py — patents is not ported. +MSG +``` + +- [ ] **Step 12: QUICK CHECK** — ruff `257`, one head (`0024`). + +--- + +### Task 6: Buy lint headroom, part B — CHECKPOINT 1 + +**Files:** +- Modify: `src/agent/agent.py`, `src/agent/main.py` + +**Interfaces:** +- Consumes: Task 3's role imports in both files. +- Produces: `src/` at **254** findings. + +**Why now and not in Task 2:** verified in rehearsal that removing these before Task 3 makes `ac2da9e`'s import-block patch conflict. + +- [ ] **Step 1: Confirm all three imports are still unused** + +```bash +.venv-test/bin/python -m ruff check src/agent/agent.py src/agent/main.py \ + --select F401 --output-format=concise +``` + +Expected: three findings — `typing.Any` and `PostRef` in `agent.py`, `sys` in `main.py`. If ruff reports fewer, an earlier task already removed one; skip that removal. + +- [ ] **Step 2: Remove them** + +In `src/agent/agent.py` delete the line `from typing import Any`, and change +`from src.agent.state import AgentState, PostRef, ThreadState` +to +`from src.agent.state import AgentState, ThreadState`. + +In `src/agent/main.py` delete the line `import sys`. + +- [ ] **Step 3: Verify both modules still import** + +```bash +.venv-test/bin/python -c "import src.agent.agent, src.agent.main; print('ok: both import')" +.venv-test/bin/python -m ruff check src/agent/agent.py src/agent/main.py --select F401 --output-format=concise +``` + +Expected: `ok: both import`, and no F401 findings. + +> Note: `ruff check src/agent/agent.py` reports one **pre-existing** `F821 Undefined name AsyncSession` at the `persist_private_profile_to_db(self, db: "AsyncSession")` string annotation, whose type is imported lazily in the function body. It is present on `cohort-db-conversations` and on `blackbird`. Leave it. + +- [ ] **Step 4: Verify the count** + +```bash +.venv-test/bin/python -m ruff check src --output-format=concise --quiet | grep -c . +``` + +Expected: `254` + +- [ ] **Step 4b: Drop `test_roles.py`'s unused `pathlib.Path` import, as its own commit** + +`46a8391`'s `test_roles.py` imports `Path` and never uses it; the F401 fails the +tests-at-zero ruff stage of the CHECKPOINT below (measured — CHECKPOINT 1 is the first +`ci.sh` run after the file lands). On blackbird the removal is part of `3a23e73`'s +lint-debt hunk; here it must land before the checkpoint. Delete the +`from pathlib import Path` line, verify +`.venv-test/bin/python -m ruff check tests` is clean, and commit with a +`Ported-from: 3a23e73 (partial)` trailer. (Task 9 Step 4's resolution then reverts to +a plain `--ours`: nothing of that hunk remains to hand-apply.) + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/agent.py src/agent/main.py +git commit -F - <<'MSG' +chore: drop three unused imports the role refactor left behind + +src/ ruff findings 257 -> 254. Deferred out of the first lint commit because +removing these before ac2da9e lands makes its import-block patch conflict: +verified that `from typing import Any`, `PostRef` and `import sys` all survive +the role work untouched. + +The pre-existing F821 on agent.py's `db: "AsyncSession"` string annotation is +left alone — it is present on cohort-db-conversations and on blackbird, and the +type is imported lazily inside the method. + +Ported-from: 3a23e73 (partial) +MSG +``` + +- [ ] **Step 6: CHECKPOINT 1 — full gate** + +```bash +./scripts/ci.sh +``` + +Expected: `==> CI passed.` with `254 findings (ceiling 260)`, a clean `0018 -> head` round trip, and coverage at or above 60%. **Do not continue past a red gate.** + +--- + +### Task 7: Cohort feed and topology — 18 picks, then CHECKPOINT 2 + +**Files:** +- Create: `src/services/conversation_feed.py`, `templates/agent/_thread_replies.html`, `tests/integration/test_conversation_feed.py`, `docs/specs/2026-08-05-conversations-cohort-scope-and-threads-design.md`, `docs/plans/2026-08-05-conversations-cohort-scope-and-threads.md` +- Modify: `src/routers/{agent_page,admin}.py`, `src/models/cohort.py`, `src/services/cohorts.py`, `templates/agent/conversations.html`, `templates/admin/cohort_topology.html`, `specs/cohort-system-v2.md`, `tests/integration/{test_agent_page,test_cohort_admin}.py`, `tests/unit/test_reachability.py` + +**Interfaces:** +- Consumes: `compute_gates` from `src/services/cohorts.py` (already on cohort); Task 3's `templates/admin/cohort_topology.html` Role column. +- Produces: `src/services/conversation_feed.py` exporting `gate_clause(gate: set[str] | None) -> ColumnElement[bool]`, `own_or_gated(gate: set[str] | None, agent_id: str) -> ColumnElement[bool]`, and `async resolve_agent_gate(db: AsyncSession, agent_id: str) -> set[str] | None`; route `GET /agent/{agent_id}/thread/{message_ts}`. + +**Two of these fix live org1 defects independent of the cohort gate:** the admin topology matrix cannot be saved at all (60×56 posts 3,528 form fields against Starlette's `max_fields=1000`), and `d2b3b21` additionally bounds a multiplicative cross-product DoS. The feed gating itself is inert while the gate is off — `gate_clause(None)` returns `true()` — and is landed now so the coming cohort flip is safe. + +- [ ] **Step 1: Cherry-pick all eighteen, in this order** + +```bash +git cherry-pick 8bc0e24 d84ce6b 0efd6a5 d2b3b21 ffad1a1 942b31b 5bf587e 230a4c0 \ + 6d94148 f77a0a2 ddb3892 0c04be6 0dd94db 44f1ad0 cc3c90f 4bc5cbe \ + a968d7a 7f6b304 +``` + +Expected: eighteen commits, no conflict. + +- [ ] **Step 2: Prove the gate is inert with isolation off** + +```bash +.venv-test/bin/python -c " +from src.services.conversation_feed import gate_clause, own_or_gated +from sqlalchemy import true +c = gate_clause(None) +assert str(c) == str(true()), str(c) +print('ok: gate_clause(None) is a no-op ->', c) +print('ok: own_or_gated(None, \"su\") ->', own_or_gated(None, 'su')) +" +``` + +Expected: `gate_clause(None)` renders as `true`. This is the mechanical proof that org1's mesh behaviour is unchanged. + +- [ ] **Step 3: Verify the topology form no longer posts per-cell markers** + +```bash +grep -c 'name="present"' templates/admin/cohort_topology.html +grep -c 'name="present_agent"' templates/admin/cohort_topology.html +grep -c 'name="present_cohort"' templates/admin/cohort_topology.html +grep -n '_TOPOLOGY_MAX_FIELDS' src/routers/admin.py +``` + +Expected: `0`, `1`, `1`, and `_TOPOLOGY_MAX_FIELDS = 50_000` defined and used in `form(max_fields=...)`. + +- [ ] **Step 4: Verify Task 3's Role column survived the template picks** + +```bash +grep -c 'a.role' templates/admin/cohort_topology.html +``` + +Expected: at least `1`. If `0`, a pick clobbered Task 3's column — resolve by re-adding the `Role` header and the `{{ a.role }}` cell. + +- [ ] **Step 5: Run the feed and admin suites** + +```bash +.venv-test/bin/python -m pytest \ + tests/integration/test_conversation_feed.py \ + tests/integration/test_cohort_admin.py \ + tests/integration/test_agent_page.py \ + tests/unit/test_reachability.py -q +``` + +Expected: all pass. + +- [ ] **Step 6: CHECKPOINT 2 — full gate** + +```bash +./scripts/ci.sh +``` + +Expected: `==> CI passed.`, ruff ~`255`, empty snapshot diff. + +--- + +### Task 8: Rate limiter core — 11 picks, 1 conflict + +**Files:** +- Modify: `src/agent/{simulation,state,agent,pi_handler}.py`, `src/config.py`, `src/services/llm.py`, `tests/unit/{test_roles,test_cohort_isolation,test_hub_budget_scheduler,test_roster_sync}.py`, `tests/integration/test_state_rebuild.py` +- Create: `tests/unit/test_hub_budget_scheduler.py`, `docs/specs/2026-08-06-hub-budget-scheduler-design.md`, `docs/plans/2026-08-06-hub-budget-scheduler.md` + +**Interfaces:** +- Consumes: `RoleSpec` from Task 3 (gains `calls_per_load_per_window: int | None`). +- Produces: `Agent.record_api_call(now: float | None = None) -> None`; `AgentState.call_times: deque[float]`; `AgentState.throttled: bool`; `Settings.llm_rate_window_seconds: int = 600`; `Settings.llm_calls_per_load_per_window: int = 8`. Tasks 13 and 14 call `record_api_call`. + +**Why it ports as one unit:** `record_api_call` is the single write point for both the lifetime counter and the sliding-window ledger. The bug it fixes is generic and severe — `_rebuild_state` restores `api_call_count` from `llm_call_logs`, so a crossed cumulative cap benches an agent permanently, *across restarts*. + +- [ ] **Step 1: Pick the first four (clean)** + +```bash +git cherry-pick 15a277e 7c6768e 5654271 09b83aa +``` + +Expected: four commits (three docs + `_agent_load`), no conflict. + +- [ ] **Step 2: Pick `9932645` and resolve two conflicts** + +```bash +git cherry-pick 9932645 +``` + +Expected: conflict in `tests/unit/test_roles.py` **only**. (`src/config.py` merges +clean — measured in the audit rehearsal: `9932645`'s hunk adds only the rate-limiter +settings block, and the `uspto_api_key` / `patentsview_api_key` block it sat next to +on blackbird arrived with the excluded `0621ef3`, so it never existed here and there +is nothing to delete. The `_guard_rate_limiter_settings` validator is not in this +commit either; it arrives with `46d3a61` in Task 9 Step 6.) + +For `tests/unit/test_roles.py`: the conflict region's incoming side contains two +scout_hub tests and a `from src.agent.roles import load_role as _load_role_real` +import as *context* — they belong to excluded commits and were never written here — +followed by the five new `tmp_path`-based rate-override tests. Keep **only** the five +rate tests (`test_role_rate_override_is_read_when_positive`, `..._defaults_to_none`, +`..._rejects_non_positive`, `..._rejects_non_int`, +`test_missing_manifest_yields_no_rate_override`); drop the scout_hub block and its +import. + +- [ ] **Step 3: Verify no USPTO settings leaked in** + +```bash +grep -c 'uspto_api_key\|patentsview_api_key' src/config.py +.venv-test/bin/python -c " +from src.config import Settings +s = Settings(secret_key='x'*40, postgres_password='x') +assert s.llm_rate_window_seconds == 600 +assert s.llm_calls_per_load_per_window == 8 +assert not hasattr(s, 'uspto_api_key') +print('ok: rate limiter settings present, USPTO settings absent') +" +git add -A && git cherry-pick --continue --no-edit +``` + +Expected: `0`, then the `ok:` line. + +- [ ] **Step 4: Pick `0929870` and resolve one conflict** + +```bash +git cherry-pick 0929870 +``` + +Expected: applies **clean** — Task 4's `_load_prompt` edit already aligned the context +blackbird's parent supplies (measured in the audit rehearsal). Verify the pick's +`agent.py` delta is exactly `import time` plus the `record_api_call` method below and +nothing else (`git show HEAD --stat -- src/agent/agent.py` → 13 insertions). If it +conflicts on a tree that diverged from this plan, keep **only** the `record_api_call` +method: + +```python + def record_api_call(self, now: float | None = None) -> None: + """Record one LLM call against both the lifetime counter and the + sliding-window ledger. + + The single write point for both. Every call site must use this rather + than bumping ``api_call_count`` directly — a site that bumps only the + counter is invisible to the rate limiter, and a site that appends only to + the ledger corrupts ``SimulationRun.total_api_calls``. + """ + self.api_call_count += 1 + self.state.call_times.append(time.time() if now is None else now) +``` + +plus the `import time` at the top if absent. Take `simulation.py` and `state.py` as-is, +then `git add -A && git cherry-pick --continue --no-edit`. + +- [ ] **Step 5: Pick the remaining five (clean)** + +```bash +git cherry-pick 6d1deed 0821372 92e4989 f5531d2 e111732 +``` + +Expected: five commits, no conflict. + +- [ ] **Step 6: Verify the ledger is consistent** + +```bash +.venv-test/bin/python -m pytest \ + tests/unit/test_hub_budget_scheduler.py tests/unit/test_cohort_isolation.py \ + tests/unit/test_roster_sync.py tests/unit/test_roles.py \ + tests/integration/test_state_rebuild.py -q +``` + +Expected: all pass. + +- [ ] **Step 7: Verify no site bumps the counter directly** + +```bash +grep -rn 'api_call_count += 1' src/ | grep -v 'def record_api_call' | grep -v 'self.api_call_count += 1$' +``` + +Expected: no output other than the line inside `record_api_call` itself. Any other hit is a site invisible to the rate limiter. + +- [ ] **Step 8: QUICK CHECK** + +--- + +### Task 9: Rate limiter completion — 4 picks, 3 conflicted + +**Files:** +- Modify: `src/agent/{main,simulation,pi_handler}.py`, `src/config.py`, `docs/specs/2026-08-06-hub-budget-scheduler-design.md`, `tests/unit/{test_hub_budget_scheduler,test_cohort_isolation,test_roles,test_email_templates,test_slack_tokens}.py`, `tests/integration/{test_agent_page,test_cohort_admin,test_proposal_review}.py` + +**Interfaces:** +- Consumes: Task 8's `record_api_call`, `call_times`, settings. +- Produces: `--budget` defaulting to `0`; the throttle back-off in `_select_next_agent`. + +- [ ] **Step 1: Pick `bc4dd3a` and drop its CLAUDE.md hunk** + +```bash +git cherry-pick bc4dd3a +``` + +Expected: conflict in `CLAUDE.md`. org1's CLAUDE.md is authored separately (spec §8), so discard blackbird's version of the hunk: + +```bash +git checkout --ours CLAUDE.md +git add CLAUDE.md +git diff --cached --stat -- CLAUDE.md # expect NO output: the file is unchanged +git add -A && git cherry-pick --continue --no-edit +``` + +- [ ] **Step 2: Verify the CLI default flipped** + +```bash +grep -n 'DEPRECATED legacy cumulative cap' src/agent/main.py +.venv-test/bin/python -c " +import inspect, src.agent.main as m +src = inspect.getsource(m.main) +assert 'typer.Option(\n 0, \"--budget\"' in src or '0, \"--budget\"' in src, src[:400] +print('ok: --budget defaults to 0') +" +git diff origin/cohort-db-conversations --stat -- CLAUDE.md # expect NO output +``` + +Expected: the deprecation help text present, `ok:` line, and **no** CLAUDE.md diff. + +- [ ] **Step 3: Pick `00e174f` (clean)** + +```bash +git cherry-pick 00e174f +``` + +- [ ] **Step 4: Pick `3a23e73` and resolve two conflicts** + +```bash +git cherry-pick 3a23e73 +``` + +Expected: conflicts in `tests/unit/test_patents.py` and `tests/unit/test_roles.py`. Also expect `src/agent/agent.py`, `src/agent/main.py`, `src/agent/message_log.py`, `src/dependencies.py`, `tests/integration/test_proposal_review.py`, `tests/unit/test_email_templates.py` and `tests/unit/test_slack_tokens.py` to apply as no-ops or trivially — Tasks 2 and 6 already took them. + +`tests/unit/test_patents.py` does not exist here and never will: + +```bash +git rm -f --ignore-unmatch tests/unit/test_patents.py +``` + +`tests/unit/test_roles.py`: **take `--ours`.** The hunk's one live edit — removing the +unused `from pathlib import Path` — already landed at Task 6 Step 4b (it had to precede +CHECKPOINT 1); its other line re-tags the `_load_role_real` import, which does not +exist here: + +```bash +git checkout --ours tests/unit/test_roles.py +.venv-test/bin/python -m ruff check tests/unit/test_roles.py # expect: All checks passed! +git add tests/unit/test_roles.py +``` + +Then take the two integration hermeticity fixes (`test_agent_page.py`, +`test_cohort_admin.py`) as-is and continue: + +```bash +git add -A && git cherry-pick --continue --no-edit +``` + +- [ ] **Step 5: Verify no patents test file exists** + +```bash +ls tests/unit/test_patents.py 2>&1 | grep -q 'No such file' && echo "ok: absent" +ls tests/live_api/test_patents_live.py 2>&1 | grep -q 'No such file' && echo "ok: absent" +test ! -e src/services/patents.py && echo "ok: no patents service" +grep -rln 'search_prior_art' src/ || echo "ok: no search_prior_art in src/" +``` + +Expected: four `ok` lines. (`tests/` does carry the *string* — as tmp-path manifest +data in `test_roles.py` and an absence assertion in `test_tool_gating.py` — and after +Task 10 so does `thread_guidance.py`'s deliberately-kept dead `_SCOUT_HUB` block. None +of that is executable patents code; do not chase it.) + +- [ ] **Step 6: Pick `46d3a61` and resolve one conflict** + +```bash +git cherry-pick 46d3a61 +``` + +Expected: conflict in `src/config.py` — caused by Task 1, not by blackbird order: +copi-prod's `audit_recipient_list` property occupies exactly the spot where `46d3a61` +adds its F4 `_guard_rate_limiter_settings` validator. Keep **both** — HEAD's property +first, then the incoming validator (this commit, not `9932645`, is where the validator +arrives). Then: + +```bash +.venv-test/bin/python -c "import src.config; print('ok: config imports')" +git add -A && git cherry-pick --continue --no-edit +``` + +- [ ] **Step 7: CHECKPOINT 3 — full gate** + +```bash +./scripts/ci.sh +``` + +Expected: `==> CI passed.`, empty snapshot diff. This is the highest-behavioural-risk checkpoint — the scheduler now paces on a sliding window instead of a cumulative cap. + +--- + +### Task 10: Role prompt completion and LLM truncation — 4 clean picks + +**Files:** +- Create: `src/agent/thread_guidance.py`, `tests/unit/{test_thread_guidance,test_llm_service}.py` +- Modify: `src/agent/{agent,pi_handler}.py`, `src/services/llm.py`, `tests/unit/test_agent_prompts.py` + +**Interfaces:** +- Consumes: Task 3's `_load_prompt`, Task 8's `record_api_call`. +- Produces: `phase4_guidance(role: str, message_count: int) -> tuple[str, str, str]` from `src/agent/thread_guidance.py`; `generate_agent_response(..., on_retry: Callable[[], None] | None = None)`. Task 13 extends the same `on_retry` contract to `generate_with_tools`. + +`683c09a`'s `_PI_LAB` strings are byte-identical to the pre-refactor `agent.py` literals and are pinned by the characterization snapshot. Its `_SCOUT_HUB` dict is dead code without the role and is kept rather than trimmed, so the file stays mergeable with blackbird. + +- [ ] **Step 1: Cherry-pick all four** + +```bash +git cherry-pick 2467229 bc40d20 683c09a 44f09be +``` + +Expected: four commits, no conflict. + +- [ ] **Step 2: Prove the pi_lab guidance strings did not change** + +```bash +.venv-test/bin/python -c " +from src.agent.thread_guidance import phase4_guidance +for n, want in ((2,'EXPLORE'), (8,'DECIDE'), (12,'MUST CONCLUDE')): + phase, guidance, instructions = phase4_guidance('pi_lab', n) + assert phase == want, (n, phase) + assert guidance and instructions +print('ok: pi_lab phases', [phase4_guidance('pi_lab', n)[0] for n in (2,8,12)]) +assert phase4_guidance('nonsense_role', 2) == phase4_guidance('pi_lab', 2) +print('ok: unknown role degrades to pi_lab') +" +.venv-test/bin/python -m pytest tests/characterization -q +``` + +Expected: both `ok:` lines and a green characterization suite — the snapshot is the real proof. + +- [ ] **Step 3: Run the new suites** + +```bash +.venv-test/bin/python -m pytest \ + tests/unit/test_thread_guidance.py tests/unit/test_llm_service.py \ + tests/unit/test_agent_prompts.py -q +``` + +Expected: all pass. + +- [ ] **Step 4: QUICK CHECK** — empty snapshot diff is the critical one here. + +--- + +### Task 11: `_post_message` suppression and the `-> bool` contract + +**Files:** +- Modify: `src/agent/simulation.py`, `tests/unit/test_simulation_logic.py` + +**Interfaces:** +- Consumes: `SimulationEngine._post_message(self, agent_id, channel, text, thread_ts=None)` currently `-> None`. +- Produces: **`_post_message(...) -> bool`** — `True` exactly when a post was recorded, `False` when the text stripped to nothing or the parent thread was deleted. Task 14 depends on this return value. + +**Why `-> bool` and not `-> str | None`:** on blackbird the contract arrives in `29fc8f1` as `-> bool` and is widened to `-> str | None` by `1b44e1c` *solely* so an `opportunity_assessments` row can store the post's `slack_ts`. org1 has no such table. Take the `bool`. + +- [ ] **Step 1: Locate the three edit points** + +```bash +grep -n 'async def _post_message' src/agent/simulation.py +grep -n 'slack_message>\\", \\"\\", text' src/agent/simulation.py +grep -n 'Skipped reply to deleted thread' src/agent/simulation.py +grep -n 'The DB is the primary store\.$' src/agent/simulation.py +``` + +Expected: four hits. Note the line numbers; the edits below are described relative to them. + +- [ ] **Step 2: Write the failing test** + +`tests/unit/test_simulation_logic.py` has **no pytest fixtures** — it uses per-class +`_engine()` helper methods. Follow that existing pattern exactly. Blackbird's own tests +for this are not reusable: they live in `TestPostMessageStripsAssessmentSidecar` and +assert on the `` sidecar, which this branch does not have. + +Append to `tests/unit/test_simulation_logic.py`: + +```python +# --------------------------------------------------------------- +# _post_message — a text that strips to nothing must be suppressed, +# and the caller must be told. +# --------------------------------------------------------------- + +class TestPostMessageSuppressesEmptyText: + def _engine(self): + from src.agent.agent import Agent + su = Agent("su", "SuBot", "Andrew Su") + # slack_clients={} puts _post_message in MOCK mode: no network, but it + # still mints a ts and appends a LogEntry, which is what we are testing. + return SimulationEngine(agents=[su], slack_clients={}), su + + @pytest.mark.asyncio + async def test_text_that_strips_to_nothing_is_suppressed(self): + engine, _su = self._engine() + + posted = await engine._post_message("su", "general", "") + + assert posted is False + assert engine.message_log._entries == [] + + @pytest.mark.asyncio + async def test_a_real_message_is_recorded_and_reports_true(self): + engine, _su = self._engine() + + posted = await engine._post_message("su", "general", "a real message") + + assert posted is True + assert len(engine.message_log._entries) == 1 + assert engine.message_log._entries[0].content == "a real message" +``` + +- [ ] **Step 3: Run it and watch it fail** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_simulation_logic.py \ + -k TestPostMessageSuppressesEmptyText -v +``` + +Expected: both FAIL. The first with `assert None is False` **and** a non-empty +`_entries` (today the empty text is recorded as a phantom row); the second with +`assert None is True`. + +- [ ] **Step 4: Change the signature and docstring** + +Replace: + +```python + ) -> None: + """Post a message to Slack and record it in the message log + DB.""" +``` + +with: + +```python + ) -> bool: + """Post a message to Slack and record it in the message log + DB. + + Returns whether a message was actually recorded — ``False`` when the + text stripped to nothing, or the reply's parent thread was found to be + deleted. In either case nothing was posted and no log entry was written, + so a caller must not count the turn, clear backoff state, or move posts + between ``interesting_posts`` and ``active_threads``. + """ +``` + +- [ ] **Step 5: Add the emptiness guard** + +Immediately **after** the line `text = re.sub(r"", "", text).strip()`, insert: + +```python + # A truncated response can strip to nothing — the whole body may have been + # tags. Slack rejects empty text anyway, but bailing here also matters for + # what happens *after* posting: without this guard _post_message still + # mints a ts and writes a LogEntry with content="" and slack_ts=None — a DB + # row with no corresponding Slack message, breaking the + # row-count-matches-Slack-message-count invariant documented below — and the + # caller still counts the turn as published even though nothing went out. + # Return before any of that: no Slack call, no minted ts, no log entry. + if not text: + logger.warning( + "[%s] Suppressed a post to #%s: text was empty after stripping the " + "slack_message tags — likely a truncated response with no real body.", + agent_id, channel, + ) + return False +``` + +> The comment is deliberately reworded. Blackbird's version is written around +> `_strip_assessment_sidecar(text)`, a call that does not exist on this branch. + +- [ ] **Step 6: Make both remaining exits explicit** + +Change the bare `return` under the `Skipped reply to deleted thread` log to `return False`. + +At the very end of `_post_message`, after `self.message_log.append(entry)` (which sits inside the per-chunk loop at 12-space indent), add at **8-space** indent: + +```python + return True +``` + +- [ ] **Step 7: Verify every exit path returns a bool** + +```bash +.venv-test/bin/python - <<'PY' +import ast, inspect, textwrap, src.agent.simulation as m +tree = ast.parse(textwrap.dedent(inspect.getsource(m.SimulationEngine._post_message))) +bare = [n.lineno for n in ast.walk(tree) if isinstance(n, ast.Return) and n.value is None] +assert not bare, f"bare `return` still present at offsets {bare}" +print("ok: no bare returns left in _post_message") +PY +``` + +(`textwrap.dedent` is load-bearing: `inspect.getsource` of a method returns indented +source, which `ast.parse` rejects.) + +Expected: `ok:` line. + +- [ ] **Step 8: Run the tests** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_simulation_logic.py -q +``` + +Expected: PASS. + +- [ ] **Step 9: Commit** + +```bash +git add src/agent/simulation.py tests/unit/test_simulation_logic.py +git commit -F - <<'MSG' +fix(sched): suppress a post that strips to nothing, and tell the caller + +A truncated response whose whole body was tags stripped to "". +_post_message had no emptiness guard, so it still minted a ts and wrote a +LogEntry with content="" and slack_ts=None — a DB row with no Slack message +behind it — while the caller counted the turn as published. Bail out before any +state changes, and log why. + +_post_message now returns bool: True exactly when a message was recorded. The +next commit makes the callers check it; without a return value the guard above +would suppress the post and leave every caller's bookkeeping claiming success. + +bool, not `str | None`: blackbird widened the return to carry the post's +slack_ts so an opportunity_assessments row could link back to it. org1 has no +such table, so the id has no consumer here. + +Ported-from: 21869e2, 29fc8f1 (partial) +Dropped: the assessment-sidecar parsing, _strip_assessment_sidecar, the +verdict/gating persistence, and the oversized-field handling — all Blackbird +product. The suppression comment is reworded accordingly: blackbird's is +written around a _strip_assessment_sidecar call this branch does not have. +MSG +``` + +- [ ] **Step 10: QUICK CHECK** + +--- + +### Task 12: The `/admin` 500 class and the unauthenticated vote-tamper hole + +**Files:** +- Modify: `src/routers/admin.py`, `src/routers/public.py`, `templates/admin/{discussions,activity_detail}.html`, `tests/characterization/{test_auth_and_admin_routes,test_public_routes}.py` + +**Interfaces:** +- Consumes: nothing new. +- Produces: no new names; two 500-class fixes and one authorization fix. + +**The security fix:** `update_proposal_vote_details` read +`if vote_obj.voter_token and token and vote_obj.voter_token != token`. A caller who simply **omits** `voter_token` gets `token is None`, which short-circuits the check — so ownership could be bypassed by leaving the field out. + +- [ ] **Step 1: Pick `73a78c3` and resolve one conflict** + +```bash +git cherry-pick 73a78c3 +``` + +Expected: applies **clean** (measured in the audit rehearsal — the `f32a83e` admin.py +context the spec predicted would block it is not needed on this tree). Verify the +`available_agents` None-guard landed in `admin_discussions` in exactly this shape +(`grep -n -A9 'available_agents = set()' src/routers/admin.py`); if the pick conflicts +instead, apply it by hand and `git add -A && git cherry-pick --continue --no-edit`: + +```python + available_agents = set() + for t in threads: + for candidate in ( + t["agent_id"], + t.get("replier"), + t["decision"].agent_a if t.get("decision") else None, + t["decision"].agent_b if t.get("decision") else None, + ): + if candidate: + available_agents.add(candidate) +``` + +Take `tests/characterization/test_auth_and_admin_routes.py` as-is. + +- [ ] **Step 2: Pick `5fb68c0` and drop two files** + +```bash +git cherry-pick 5fb68c0 +``` + +Expected: conflicts in `src/agent/simulation.py` and `tests/integration/test_opportunity_assessment_persistence.py`. + +Both are dropped — the `simulation.py` hunk is the assessment-persist fix, and the test file is Blackbird product: + +```bash +git checkout --ours src/agent/simulation.py +git add src/agent/simulation.py +git rm -f --ignore-unmatch tests/integration/test_opportunity_assessment_persistence.py +git add -A && git cherry-pick --continue --no-edit +``` + +Keep `src/routers/public.py`, `src/routers/admin.py`, both templates, and both characterization tests. + +- [ ] **Step 3: Verify the vote-tamper hole is closed** + +```bash +grep -n 'voter_token and vote_obj.voter_token != token' src/routers/public.py +grep -c 'voter_token and token and' src/routers/public.py +``` + +Expected: the first grep hits (the fixed form); the second prints `0` (the vulnerable form is gone). + +- [ ] **Step 4: Verify no assessment code leaked into `simulation.py`** + +```bash +grep -c '_persist_assessment\|OpportunityAssessment\|assessment_json' src/agent/simulation.py +ls tests/integration/test_opportunity_assessment_persistence.py 2>&1 | grep -q 'No such file' && echo "ok: absent" +``` + +Expected: `0`, then `ok: absent`. + +- [ ] **Step 5: Verify the null-sender guards** + +```bash +grep -c 'if agent_id %}' templates/admin/activity_detail.html || true +grep -c 'unknown sender' templates/admin/activity_detail.html templates/admin/discussions.html +``` + +Expected: `(unknown sender)` present in both templates. + +- [ ] **Step 6: Run the characterization suites** + +```bash +.venv-test/bin/python -m pytest \ + tests/characterization/test_auth_and_admin_routes.py \ + tests/characterization/test_public_routes.py -q +``` + +Expected: all pass. + +- [ ] **Step 7: QUICK CHECK** + +--- + +### Task 13: Complete the truncation fix for `generate_with_tools` + +**Files:** +- Modify: `src/services/llm.py`, `src/agent/simulation.py` + +**Interfaces:** +- Consumes: Task 10's `on_retry` contract on `generate_agent_response`; Task 8's `record_api_call`. +- Produces: `generate_with_tools(..., on_retry: Callable[[], None] | None = None)`, with both internal retry sites re-checking `stop_reason`. + +**Why this is not optional:** `generate_with_tools` is the function **phase-4 thread replies use** — org1's entire product. `44f09be` (Task 10) fixed only `generate_agent_response`. Without this, one of the two retry sites never re-checks `stop_reason` at all, so a phase-4 reply truncated after doubling `max_tokens` loses its closing `` silently, and the rate limiter undercounts every retried phase-4 turn. + +- [ ] **Step 1: Confirm the gap** + +```bash +grep -n 'async def generate_with_tools' src/services/llm.py +grep -c 'on_retry' src/services/llm.py +grep -c 'Response still truncated' src/services/llm.py +``` + +Expected: `generate_with_tools` found; `on_retry` count reflects only `generate_agent_response`; one existing `logger.warning("Response still truncated after retry ...")` inside `generate_with_tools`. + +- [ ] **Step 2: Add the parameter and document it** + +Add `on_retry: Callable[[], None] | None = None,` to `generate_with_tools`'s signature (after `log_meta`), and append to its docstring: + +``` + ``on_retry``, same contract as ``generate_agent_response``'s: it fires + once — synchronously, before this returns — exactly when one of this + function's two internal max_tokens retries (the "final text" branch's, + or the max-tool-rounds fallback's; at most one runs per call) actually + makes a second API call. A caller that books one call against a rate + limiter for this whole turn (e.g. ``Agent.record_api_call``) should pass + that callable here so a retried turn is booked as the two real API calls + it made, not one. Optional and additive: omitting it changes nothing. +``` + +- [ ] **Step 3: Fire the hook at both retry sites** + +After **each** of the two `retry_msg = client.messages.create(...)` calls inside `generate_with_tools` (note: not awaited — this function uses the sync client), insert: + +```python + # Second real, billed API call for what the caller booked as + # one turn — fire the caller's own accounting hook (if any). + if on_retry is not None: + on_retry() +``` + +(match the surrounding indentation at each site). + +- [ ] **Step 4: Make both sites log loudly on a still-truncated retry** + +Replace the existing `logger.warning("Response still truncated after retry (%d tokens)", ...)` with: + +```python + agent_id = (log_meta or {}).get("agent_id", "?") + phase = (log_meta or {}).get("phase", "?") + logger.error( + "Response still truncated after 2x max_tokens retry " + "(model=%s agent=%s phase=%s retry_max_tokens=%d " + "out_tok=%d) — returning the truncated text; anything " + "the model emits last (e.g. a closing tag) may be " + "missing from it.", + model, agent_id, phase, retry_max, + retry_msg.usage.output_tokens, + ) +``` + +And at the **max-tool-rounds fallback** site — which never re-checked `stop_reason` at all — add the whole check: + +```python + if retry_msg.stop_reason == "max_tokens": + # This retry site never re-checked stop_reason before this fix: a + # still-truncated response after exhausting max_tool_rounds AND + # doubling max_tokens passed silently. + agent_id = (log_meta or {}).get("agent_id", "?") + phase = (log_meta or {}).get("phase", "?") + logger.error( + "Response still truncated after 2x max_tokens retry " + "(model=%s agent=%s phase=%s retry_max_tokens=%d " + "out_tok=%d) — returning the truncated text; anything " + "the model emits last (e.g. a closing tag) may be " + "missing from it.", + model, agent_id, phase, retry_max, retry_msg.usage.output_tokens, + ) +``` + +- [ ] **Step 5: Wire the phase-4 call site** + +In `src/agent/simulation.py`, find the `generate_with_tools(...)` call whose `log_meta` has `"phase": "thread_reply"` and add: + +```python + on_retry=agent.record_api_call, +``` + +- [ ] **Step 6: Verify both retry sites are covered** + +```bash +.venv-test/bin/python - <<'PY' +import inspect, src.services.llm as m +src = inspect.getsource(m.generate_with_tools) +assert src.count("if on_retry is not None:") == 2, src.count("if on_retry is not None:") +assert src.count("Response still truncated after 2x max_tokens retry") == 2 +assert "logger.warning(\n \"Response still truncated after retry" not in src +print("ok: both retry sites fire on_retry and log at ERROR") +PY +grep -n 'on_retry=agent.record_api_call' src/agent/simulation.py +``` + +Expected: the `ok:` line, and at least one `on_retry=agent.record_api_call` in `simulation.py`. + +- [ ] **Step 7: Run the LLM suite** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_llm_service.py -q +``` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add src/services/llm.py src/agent/simulation.py +git commit -F - <<'MSG' +fix(llm): finish the truncation fix — generate_with_tools had the same defects + +generate_with_tools has two internal max_tokens retry sites. Only one re-checked +stop_reason, and neither reported the extra call, so a retried turn booked as +one call against a limiter that had already counted it once. It is the function +phase-4 thread replies use, which on this deployment is the whole product: a +reply truncated after doubling max_tokens lost its closing with +no trace in the logs. + +Both sites now fire the caller's on_retry hook and log at ERROR with the model, +agent, phase and output-token count. simulation.py's thread_reply call site +passes record_api_call, so the sliding-window limiter paces on real API calls. + +Ported-from: f32a83e (partial) +Dropped: B1/B2/B3 — the /admin/assessments triage-queue run scoping, the +derisking_milestones column, and the assessments.html styling. All Blackbird +product. +MSG +``` + +- [ ] **Step 9: QUICK CHECK** + +--- + +### Task 14: Make `_post_message`'s callers check its return + +**Files:** +- Modify: `src/agent/simulation.py`, `tests/unit/test_simulation_logic.py` + +**Interfaces:** +- Consumes: Task 11's `_post_message(...) -> bool`. +- Produces: no new names — four call sites that no longer book a turn for a message nobody saw. + +**Why:** Task 11 made `_post_message` report suppression; on this branch **no** caller checks it. (On blackbird the phase-5 "new top-level post" branch was guarded by `29fc8f1`'s caller-side hunk — which Task 11 deliberately does not port, taking only the return contract — so here that site is unguarded too; found in the audit rehearsal.) All four sites counted the turn, cleared pending-reply/backoff state, and moved posts between `interesting_posts` and `active_threads` regardless. Worst on phase 4, which is org1's main path. + +- [ ] **Step 1: Find the four unguarded call sites** + +```bash +grep -n 'await self._post_message(' src/agent/simulation.py +``` + +Expected: exactly four hits — the phase-4 reply, the phase-5 private-channel flat follow-up, the phase-5 thread-creating reply, and the phase-5 new top-level post. + +- [ ] **Step 2: Write the failing structural check** + +Driving the real phase-4 handler needs a thread, an LLM stub and a Slack transport — +too much scaffolding to specify blind, and the resulting test would assert less than a +direct structural check does. The gate for this task is therefore an AST assertion that +**no `_post_message` call discards its result**, which is exactly the defect. + +Save as `/tmp/check_post_message_callers.py`: + +```python +import ast +import inspect + +import src.agent.simulation as m + +tree = ast.parse(inspect.getsource(m.SimulationEngine)) +discarded = [] +for node in ast.walk(tree): + # An Expr whose value is an Await of a _post_message call = result thrown away. + if isinstance(node, ast.Expr) and isinstance(node.value, ast.Await): + call = node.value.value + if isinstance(call, ast.Call) and getattr(call.func, "attr", "") == "_post_message": + discarded.append(node.lineno) +if discarded: + raise SystemExit( + f"FAIL: {len(discarded)} _post_message call site(s) discard the result, " + f"at source offsets {discarded}" + ) +print("ok: every _post_message result is consumed") +``` + +- [ ] **Step 3: Run it and watch it fail** + +```bash +.venv-test/bin/python /tmp/check_post_message_callers.py +``` + +Expected: `FAIL: 4 _post_message call site(s) discard the result, at source offsets [...]`. +Note the four offsets — the phase-4 reply, the phase-5 private-channel flat follow-up, +the phase-5 thread-creating reply, and the phase-5 new top-level post. If the count is +not 4, reconcile against `grep -n 'await self._post_message(' src/agent/simulation.py` +before editing. + +- [ ] **Step 4: Guard each of the four sites** + +At each site, capture the return and put every side effect behind it: + +```python + posted = await self._post_message(...) + if not posted: + logger.info( + "[%s] Suppressed post in #%s — not counted, nothing persisted", + agent.agent_id, channel, + ) + else: + # ... every existing side effect, unchanged, indented one level +``` + +The side effects that must move inside the `else`: `agent.message_count += 1`, the +`interesting_posts` filter, the `active_threads[...] = ThreadState(...)` assignment, any +pending-reply or backoff reset, and the existing success `logger.info`. At the phase-4 +site that includes `thread.has_pending_reply = False`, the +`funding_reject_count`/`empty_response_count` resets and the +`_check_thread_outcome` await; at the new-post site it is the count plus the +tagged/untagged success logs. + +- [ ] **Step 5: Run the structural check again** + +```bash +.venv-test/bin/python /tmp/check_post_message_callers.py +``` + +Expected: `ok: every _post_message result is consumed`. + +- [ ] **Step 6: Run the tests** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_simulation_logic.py tests/unit/test_cohort_isolation.py -q +``` + +Expected: PASS. In particular Task 11's `TestPostMessageSuppressesEmptyText` must still +pass — the guards must not have changed `_post_message` itself. + +- [ ] **Step 7: Commit** + +```bash +git add src/agent/simulation.py tests/unit/test_simulation_logic.py +git commit -F - <<'MSG' +fix(agent): a suppressed post must not count as a turn + +_post_message returns False when nothing reached Slack, and no caller checked +it: the phase-4 reply site, both phase-5 reply branches (private-channel flat +follow-up, thread-creating reply) and the phase-5 new top-level post branch +counted the turn, cleared pending-reply and backoff state, and moved posts +between interesting_posts and active_threads for a message nobody ever saw. +(On blackbird the new-post branch was guarded by 29fc8f1's caller hunk; the +previous commit here took only that commit's return contract.) All four now +check, and skip every one of those side effects when suppressed, leaving +state exactly as if the turn had not been attempted. + +Phase 4 is the site that matters most here: collaboration replies are this +deployment's whole product. + +Ported-from: e116feb, 29fc8f1 (partial) +Dropped: the _extract_assessment_json newest-first rework and the three-way +sidecar outcome logging at the phase-5 call site. Both are Blackbird product. +MSG +``` + +- [ ] **Step 8: QUICK CHECK** + +--- + +### Task 15: Refuse a phase-5 response with no `action` + +**Files:** +- Modify: `src/agent/simulation.py` + +**Interfaces:** +- Consumes: the phase-5 handler's `action_data` dict. +- Produces: no new names. + +**Why:** cohort reads `action = action_data.get("action", "new_post")`, so a response whose JSON omits `action` silently becomes a top-level post — with whatever `post_type` and channel happened to parse. That is a latent defect on org1 today. + +- [ ] **Step 1: Confirm the current default** + +```bash +grep -n 'action_data.get("action"' src/agent/simulation.py +``` + +Expected: `action = action_data.get("action", "new_post")`. + +- [ ] **Step 2: Replace the default with a refusal** + +```python + # A missing `action` is an unparseable response, not a license to + # post something anyway — defaulting to "new_post" here is what lets + # a malformed action dict fall through into posting to #general with + # an empty post_type instead of being rejected outright. + action = action_data.get("action") + if not action: + logger.warning( + "[%s] Phase 5: parsed JSON had no 'action' field — " + "treating as unparseable", + agent.agent_id, + ) + return +``` + +- [ ] **Step 3: Verify no implicit default remains** + +```bash +grep -c 'action_data.get("action", "new_post")' src/agent/simulation.py +grep -n 'action_data.get("action")' src/agent/simulation.py +``` + +Expected: `0`, then one hit for the bare form. + +- [ ] **Step 4: Run the phase-5 tests** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_simulation_logic.py tests/characterization -q +``` + +Expected: PASS, no snapshot movement. + +- [ ] **Step 5: Commit** + +```bash +git add src/agent/simulation.py +git commit -F - <<'MSG' +fix(phase5): a response with no `action` is unparseable, not a new post + +action_data.get("action", "new_post") turned a malformed phase-5 response into +a top-level post carrying whatever post_type and channel happened to parse. +Refuse the turn and log it instead. + +Ported-from: 1b44e1c (partial) +Dropped: the fenced-sidecar hijack guard and the downstream +verdict-persistence fixes (Blackbird product), and the phase-5 max_tokens +1000 -> 2500 increase. That ceiling was sized for scout_hub's eleven-section +assessment artifact plus its JSON sidecar; it is unconditional across all +roles, and on this deployment it is a 2.5x output-token increase with nothing +to spend it on. +MSG +``` + +- [ ] **Step 6: QUICK CHECK** + +--- + +### Task 16: Post-type machinery — 5 picks, 1 conflict, 1 trimmed + +**Files:** +- Create: `src/agent/post_types.py`, `tests/unit/{test_post_types,test_lab_directory_ordering}.py` +- Modify: `src/agent/{simulation,roles,agent}.py`, `tests/unit/test_roles.py` + +**Interfaces:** +- Consumes: Task 3's `RoleSpec`, `load_role`. +- Produces: `src/agent/post_types.py` exporting `PostTypeSpec`, `CANONICAL: dict[str, PostTypeSpec]`, `DEFAULT_POST_TYPES: tuple[PostTypeSpec, ...]`, `FUNDING_POST_TYPES: frozenset[str]`, `LEGACY_POST_TYPE_ALIASES: dict[str, str]`, `resolve_post_type_name(name) -> str`, `parse_post_types(raw, *, role) -> tuple[PostTypeSpec, ...]`, `eligible_targets(...)`, `available_for(...)`, `render_menu(...)`; `RoleSpec.post_types`; `Agent.build_phase5_prompt(..., post_type_menu: str | None = None)`; **`SimulationEngine.refresh_lab_directories() -> None`** (Task 17 calls it). + +**This is inert on org1.** `f2cbfe9` substitutes `{post_type_menu}` with `str.replace`, and org1's `prompts/phase5-new-post.md` carries no such token, so nothing renders. **No enforcement call is ported** — `66948dc` is excluded, so no post is ever judged against the vocabulary. + +`TERMINAL_POST_TYPES` is deliberately absent: it arrives with the excluded `96c6243`, and nothing on this branch references it. + +- [ ] **Step 1: Pick `3fd8a91` and resolve one conflict** + +```bash +git cherry-pick 3fd8a91 +``` + +Expected: applies **clean** on this tree (measured in the audit rehearsal — Tasks +11/13/14/15's simulation.py edits do not touch `3fd8a91`'s hunks). The pick itself +lands the whole restructure; the six points below describe what must now be true — +verify each landed (the directory becomes a derived product of the gate, refreshed on +the gate's own cadence). If the pick conflicts on a diverged tree, apply them by hand. + +**1a.** Add the public alias next to `_build_lab_directories`: + +```python + # Public alias. `_build_lab_directories` is called from three places whose + # ordering relative to the cohort gate is the whole bug this name documents: + # it must run AFTER _recompute_allowed_sender_ids, never before. + def refresh_lab_directories(self) -> None: + """Rebuild every agent's lab directory against its CURRENT gate.""" + self._build_lab_directories() +``` + +**1b.** In `start()`, **delete** the `self._build_lab_directories()` call that sits before +`await self._load_pi_mappings()`, and **add** after `await self._recompute_allowed_sender_ids()`: + +```python + # AFTER the gate, never before: the filter inside reads + # agent.allowed_sender_ids, which is None until the line above runs. + self.refresh_lab_directories() +``` + +**1c.** In the roster-sync no-change branch, replace the `if role_changed: self._build_lab_directories()` / `await self._recompute_allowed_sender_ids()` pair with: + +```python + # Recompute the gate FIRST; the directory rebuild below reads it. + # _recompute_allowed_sender_ids refreshes the directory itself + # whenever the gate signature moves, so only a role change needs + # an unconditional rebuild here. + await self._recompute_allowed_sender_ids() + if role_changed: + self.refresh_lab_directories() + return +``` + +**1d.** In the roster-sync membership-change path, **delete** the +`self._build_lab_directories()` call that precedes +`self.message_log.set_bot_name_map(...)`, and add `self.refresh_lab_directories()` +immediately after the trailing `await self._recompute_allowed_sender_ids()`. + +**1e.** In `_recompute_allowed_sender_ids`, add `self.refresh_lab_directories()` after +**both** `self._disable_all_gates()` calls — the `not settings.cohort_isolation_enabled` +path and the preflight-refusal path. **This is the pair that makes the change a true +no-op on org1**, where isolation is off and the disabled path is the only one taken. + +**1f.** At the end of `_recompute_allowed_sender_ids`, after `self._apply_cohort_gate_to_state()`: + +```python + # The directory is derived from the gate, so it is refreshed on the same + # cadence. Cheap: it re-reads in-memory profiles, no I/O. + self.refresh_lab_directories() +``` + +Verify: + +```bash +grep -n 'def refresh_lab_directories' src/agent/simulation.py +grep -A1 '_disable_all_gates()' src/agent/simulation.py | grep -c refresh_lab_directories # expect 2 +.venv-test/bin/python -m pytest tests/unit/test_lab_directory_ordering.py -q +``` + +- [ ] **Step 2: Pick the next two (clean)** + +```bash +git cherry-pick f231bc8 20065e1 +``` + +Expected: two commits, no conflict. (`dc371af` and `f2cbfe9`, the other two of this +task's five, have their own steps below.) + +- [ ] **Step 3: Pick `dc371af` and drop the scout_hub pieces** + +```bash +git cherry-pick dc371af +``` + +Expected: conflicts in `prompts/roles/scout_hub/role.toml` and `tests/unit/test_roles.py`. + +The role.toml does not exist here and must not be created: + +```bash +git rm -f --ignore-unmatch prompts/roles/scout_hub/role.toml +``` + +For `tests/unit/test_roles.py`: keep the **four** `tmp_path`-based `post_types` tests +(`test_missing_manifest_yields_default_post_types`, +`test_manifest_post_types_are_parsed`, `test_manifest_unknown_post_type_is_dropped`, +`test_malformed_toml_still_yields_default_post_types`). **Drop +`test_scout_hub_declares_its_two_post_types` and +`test_scout_hub_cannot_post_a_cross_lab_idea`** — both call `load_role("scout_hub")` +and need the `role.toml` just removed. + +```bash +git add -A && git cherry-pick --continue --no-edit +``` + +- [ ] **Step 4: Take `f2cbfe9` without its four blackbird-prompt tests** + +```bash +git cherry-pick -n f2cbfe9 +``` + +Four of its new `tests/unit/test_agent_prompts.py` tests assert the rendered phase-5 +prompt **contains** the menu — true only when `prompts/phase5-new-post.md` carries the +`{post_type_menu}` token, which arrives with the excluded `0e1ac52`. On org1's frozen +prompt they can never pass (found in the audit rehearsal). Delete +`test_phase5_menu_defaults_to_the_unfiltered_pi_lab_set`, +`test_phase5_default_menu_is_the_agents_own_role_not_pi_lab`, +`test_phase5_menu_uses_the_caller_supplied_text_when_given` and +`test_phase5_menu_survives_funding_only_surgery`, keeping +`test_phase5_menu_token_is_always_substituted` and +`test_phase5_default_menu_never_prints_an_empty_enumeration` — the two that state +org1's truth. Then: + +```bash +.venv-test/bin/python -m pytest tests/unit/test_agent_prompts.py -q # all pass +git add -A +git commit -F - <<'MSG' +feat(agent): substitute {post_type_menu} in the phase-5 prompt + +build_phase5_prompt renders the role's declared post types through +render_menu() and substitutes the {post_type_menu} token via str.replace — +inert on a prompt that carries no token, which is exactly org1's case: its +prompts/phase5-new-post.md is frozen and tokenless, so nothing renders and +nothing changes. The mechanism lands so the coming cohort flip can enable a +menu without another port. + +Ported-from: f2cbfe9 (partial) +Dropped: the four menu-presence tests +(test_phase5_menu_defaults_to_the_unfiltered_pi_lab_set, +test_phase5_default_menu_is_the_agents_own_role_not_pi_lab, +test_phase5_menu_uses_the_caller_supplied_text_when_given, +test_phase5_menu_survives_funding_only_surgery) — they assert the menu +renders, which needs the {post_type_menu} token only the excluded 0e1ac52 +adds. Kept the token-absence and empty-enumeration guards, the two that hold +on a frozen prompt. +MSG +``` + +- [ ] **Step 5: Prove the machinery is inert** + +```bash +grep -c '{post_type_menu}' prompts/phase5-new-post.md +git diff origin/cohort-db-conversations --stat -- prompts/phase5-new-post.md +grep -rn 'TERMINAL_POST_TYPES\|_post_type_rejection\|available_post_types' src/ || echo "ok: no enforcement anywhere" +ls prompts/roles/ 2>&1 | grep -q 'No such file' && echo "ok: no prompts/roles tree" || ls prompts/roles/ +``` + +Expected: `0` occurrences of the token, **no diff** to the base prompt, `ok: no enforcement anywhere`, and no `prompts/roles/` tree. + +- [ ] **Step 6: Verify the mesh-safety properties hold** + +```bash +.venv-test/bin/python -c " +from src.agent.post_types import (CANONICAL, DEFAULT_POST_TYPES, available_for, + resolve_post_type_name) +# The legacy alias is what keeps a prompt that still says 'idea' working. +assert resolve_post_type_name('idea') == 'idea_crosslab' +names = {s.name for s in DEFAULT_POST_TYPES} +# org1's phase5 prompt enum must be fully covered by pi_lab's declared set. +enum = {'introduction','paper','help_wanted','idea_crosslab','funding_collab'} +assert enum <= names, enum - names +# In a hubless mesh, 'pitch' drops automatically: no scout_hub agent exists. +roles = {'su':'pi_lab','wiseman':'pi_lab'} +avail = {s.name for s in available_for(DEFAULT_POST_TYPES, gate=None, + roles_by_agent=roles, self_id='su', funding_only=False)} +assert 'pitch' not in avail, avail +assert 'idea_crosslab' in avail +print('ok: mesh set =', sorted(avail)) +" +``` + +Expected: `ok: mesh set = ['funding_collab', 'help_wanted', 'idea_crosslab', 'introduction', 'paper']` + +- [ ] **Step 7: Run the suites** + +```bash +.venv-test/bin/python -m pytest \ + tests/unit/test_post_types.py tests/unit/test_roles.py \ + tests/unit/test_lab_directory_ordering.py tests/unit/test_agent_prompts.py -q +``` + +Expected: all pass. + +- [ ] **Step 8: QUICK CHECK** — empty snapshot diff is critical: `f2cbfe9` touches the phase-5 prompt builder. + +--- + +### Task 17: `parse_post_types` dedupe and directory refresh on gate failure + +**Files:** +- Modify: `src/agent/post_types.py`, `src/agent/simulation.py` + +**Interfaces:** +- Consumes: Task 16's `parse_post_types`, and `3fd8a91`'s restructured `_recompute_allowed_sender_ids`. +- Produces: no new names. + +- [ ] **Step 1: Write the failing test for the dedupe** + +Add to `tests/unit/test_post_types.py`: + +```python +def test_duplicate_post_type_entries_collapse_last_wins(caplog): + """Two [[post_types]] entries for one name must yield ONE spec — the later + one — not two contradictory entries.""" + raw = [ + {"name": "idea_crosslab", "targets": ["pi_lab"]}, + {"name": "paper"}, + {"name": "idea_crosslab", "targets": []}, + ] + with caplog.at_level("WARNING"): + out = parse_post_types(raw, role="probe") + names = [s.name for s in out] + assert names == ["idea_crosslab", "paper"], names # first-occurrence order + assert out[0].targets == frozenset() # last wins + assert any("duplicate post_types entry" in r.message for r in caplog.records) +``` + +Add `from src.agent.post_types import parse_post_types` to the imports if absent. + +- [ ] **Step 2: Run it and watch it fail** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_post_types.py -k duplicate -v +``` + +Expected: FAIL — currently `parse_post_types` appends to a list, so `names` is `['idea_crosslab', 'paper', 'idea_crosslab']`. + +- [ ] **Step 3: Switch `kept` from a list to a dict** + +In `src/agent/post_types.py`, change `kept: list[PostTypeSpec] = []` to: + +```python + # A dict, not a list: a later `[[post_types]]` entry for a name already + # seen replaces the earlier one (last wins) rather than appending a second, + # contradictory line to the rendered menu. Re-assigning an existing key does + # not move it, so declaration order is still the position of the FIRST + # occurrence of each name — stable between turns. + kept: dict[str, PostTypeSpec] = {} +``` + +Replace the `kept.append(PostTypeSpec(...))` block with: + +```python + if base.name in kept: + logger.warning( + "[post_types] %s: duplicate post_types entry for %r — the " + "later one wins", + role, base.name, + ) + kept[base.name] = PostTypeSpec( + name=base.name, emoji=base.emoji, label=base.label, + when_to_use=base.when_to_use, targets=targets, + ) +``` + +and `return tuple(kept)` with `return tuple(kept.values())`. + +- [ ] **Step 4: Run the test** + +```bash +.venv-test/bin/python -m pytest tests/unit/test_post_types.py -q +``` + +Expected: PASS. + +- [ ] **Step 5: Make the directory refresh survive a failed membership query** + +In `src/agent/simulation.py`'s `_recompute_allowed_sender_ids`, find the early `return` +taken when the cohort-membership query raises (it keeps the previous gates rather than +failing the tick). Insert immediately **before** that `return`: + +```python + # The gates from the last successful tick are kept above (see the + # docstring). But the directory is DERIVED from those gates, so a + # gate that is correct-but-stale makes a directory rebuilt from it + # correct-but-stale too — which is strictly better than leaving it + # absent. Without this, a newly-added agent whose gate isn't + # reflected in any directory yet gets _lab_directory = None for the + # rest of this failed tick, and existing agents' directories omit + # it until the next successful sync. + self.refresh_lab_directories() + return +``` + +**Do not** also take `10d598f`'s other `simulation.py` hunk, which adds +`"post_type_rejections": dict(sorted(self._post_type_rejections.items()))` to +`cohort_topology_snapshot`. That counter is populated only by the post-type enforcement +this branch does not port, so the attribute does not exist and the snapshot would raise. + +- [ ] **Step 6: Verify** + +```bash +.venv-test/bin/python -m pytest \ + tests/unit/test_post_types.py tests/unit/test_lab_directory_ordering.py \ + tests/unit/test_cohort_isolation.py -q +``` + +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +git add src/agent/post_types.py src/agent/simulation.py tests/unit/test_post_types.py +git commit -F - <<'MSG' +fix(post_types): dedupe duplicate entries; keep the directory on a gate failure + +Two [[post_types]] entries for the same name produced two contradictory specs +while any by-name lookup silently kept only the last. parse_post_types now +dedupes by name (last wins, first-occurrence order preserved) with a WARNING. + +_recompute_allowed_sender_ids now refreshes the lab directories even when the +membership query raises, so a stale-but-correct gate does not leave a directory +absent rather than merely stale. Pairs with the directory-after-gate +reordering. + +Ported-from: 0a57e41, 10d598f (partial) +Dropped: the body-mention rejection, the skip-backoff pre-reset capture, the +tagged_agent near-miss normalisation, the _post_type_rejections counter and its +admin banner row, and every prompt hunk. All of those exist only to serve the +post-type enforcement this branch deliberately does not enable. +MSG +``` + +- [ ] **Step 8: QUICK CHECK** + +--- + +### Task 18: Land the post-type design doc + +**Files:** +- Create: `docs/specs/2026-08-06-role-topology-post-type-gating-design.md` + +**Interfaces:** +- Consumes: nothing. +- Produces: the document `src/agent/post_types.py`'s module docstring cites. + +- [ ] **Step 1: Take the design doc at its final blackbird state** + +```bash +git checkout origin/blackbird -- docs/specs/2026-08-06-role-topology-post-type-gating-design.md +``` + +- [ ] **Step 2: Confirm the draft prompt tree did NOT come with it** + +```bash +git status --short +ls docs/specs/2026-08-06-post-type-gating-prompts-draft 2>&1 | grep -q 'No such file' \ + && echo "ok: no draft prompt tree" +``` + +Expected: only the one design doc staged, and `ok: no draft prompt tree`. That directory contains blackbird's full prompt set including scout_hub and must never land here. + +- [ ] **Step 3: Add an org1 preamble recording that enforcement is not enabled** + +Insert immediately after the document's `**Branch:**` line: + +```markdown +> **org1 note (2026-08-10).** This branch ports the *machinery* described below — +> `src/agent/post_types.py`, the `role.toml` `post_types` key, and the +> `{post_type_menu}` substitution — but **not** the enforcement in §"Layer 3" or the +> phase-5 rejection call. `66948dc` is deliberately excluded: org1 runs a mesh with +> `cohort_isolation_enabled=False`, where layers 2 and 3 are inert and layer 1 buys +> nothing, and its `prompts/phase5-new-post.md` carries no `{post_type_menu}` token, so +> no menu renders. Enable enforcement when cohorts are turned on, together with a +> purpose-built org1 prompt variant, as its own change with its own measurement. See +> `docs/specs/2026-08-10-org1-parity-design.md` §7.1. +``` + +- [ ] **Step 4: Verify the citation now resolves** + +```bash +grep -n 'role-topology-post-type-gating-design' src/agent/post_types.py +ls docs/specs/2026-08-06-role-topology-post-type-gating-design.md +``` + +Expected: the docstring citation, and the file present. + +- [ ] **Step 5: Commit** + +```bash +git add docs/specs/2026-08-06-role-topology-post-type-gating-design.md +git commit -F - <<'MSG' +docs(spec): land the post-type gating design, without the draft prompt tree + +src/agent/post_types.py's module docstring cites this document. Ported at its +final state (d6bf5d7 as amended by a187a1d, 31cb20c and 454fa86) so the +citation resolves, with a preamble recording that this branch takes the +machinery and not the enforcement. + +Deliberately excludes docs/specs/2026-08-06-post-type-gating-prompts-draft/, +which is blackbird's full prompt set including the scout_hub persona. +MSG +``` + +- [ ] **Step 6: QUICK CHECK** + +--- + +### Task 19: Final gate and branch verification + +**Files:** none modified. + +**Interfaces:** +- Consumes: everything. +- Produces: a verified branch ready for review and a migration window. + +- [ ] **Step 1: The decisive assertion — the snapshot never moved** + +```bash +git diff origin/cohort-db-conversations -- tests/characterization/__snapshots__/ +``` + +Expected: **no output.** This is the mechanical proof that org1's agent behaviour is unchanged. If it prints anything, a prompt changed; find which task and revert it. + +- [ ] **Step 2: No base prompt changed, and only `identity.md` was added** + +```bash +git diff origin/cohort-db-conversations --stat -- prompts/ +``` + +Expected: exactly two lines — `prompts/identity.md | 3 +` and +`prompts/daily_audit.md | 17 ...`. The latter is copi-prod's `9ab5555` arriving via +Task 1's merge (see Task 3 Step 3); the four agent-behaviour prompts must not appear. + +- [ ] **Step 3: No Blackbird product landed** + +```bash +for p in prompts/roles prompts/specialists src/services/patents.py \ + src/services/blackbird_rubric.py src/agent/specialists.py \ + src/models/opportunity.py templates/admin/assessments.html; do + test -e "$p" && echo "LEAKED: $p" || echo "ok absent: $p" +done +ls alembic/versions/ | tail -3 +grep -rln 'opportunity_assessments\|OpportunityAssessment\|blackbird_rubric\|consult_specialist' src/ tests/ || echo "ok: no Blackbird product references" +grep -rln 'search_prior_art' src/ | grep -v 'thread_guidance.py' || echo "ok: search_prior_art only in kept dead strings" +``` + +Expected: seven `ok absent:` lines, the last migration is `0024_add_agent_role.py`, +`ok: no Blackbird product references`, and `ok: search_prior_art only in kept dead +strings` (`thread_guidance.py`'s `_SCOUT_HUB` block is deliberately-kept dead code — +Task 10 — and the string also appears as tmp-path test data in `test_roles.py` / +`test_tool_gating.py`). + +- [ ] **Step 4: Still a superset of production** + +```bash +git diff --diff-filter=A --name-only HEAD origin/copi-prod +``` + +Expected: exactly `templates/onboarding/add_texts.html` and `templates/onboarding/complete.html` — the same two as Task 1, and nothing more. + +- [ ] **Step 5: Full gate** + +```bash +./scripts/ci.sh +``` + +Expected: `==> CI passed.` with a single alembic head `0024`, a clean `head -> 0018 -> head` round trip, `tests/` at zero ruff findings, `src/` at roughly **256** (ceiling 260), and coverage at or above 60%. + +- [ ] **Step 6: Confirm the commit count and read the log** + +```bash +git rev-list --count origin/cohort-db-conversations..HEAD +git log --oneline origin/cohort-db-conversations..HEAD | cat +``` + +Expected: **~76** commits (76 when Step 6 first runs; the count grows by one when this plan's own final counts-correction commit lands) — 6 docs commits (four spec/plan commits, the +audit-amendments commit, and the Step-4b sequencing correction), 66 port-side commits +(the merge, 52 verbatim picks, 10 partial/hand-applied, Task 3 Step 6b's test repair, +Task 6 Step 4b's lint fix), and copi-prod's 4 unique commits, which the merge brings +into `rev-list`'s count. Read the list: every hand-applied commit should carry a +`Ported-from:` trailer. + +- [ ] **Step 7: Verify every hand-applied commit is attributed** + +```bash +git log origin/cohort-db-conversations..HEAD --format='%H %s%n%b' \ + | grep -c 'Ported-from:' +``` + +Expected: **`11`** trailer lines, from Tasks 2, 4, 5, 6 (Steps 4b and 5 — two +commits), 11, 13, 14, 15, 16 (the `f2cbfe9` partial) and 17. Between them they cite +twelve blackbird shas — `3a23e73` three times (Tasks 2 and 6's two commits, which +pre-apply parts of a commit Task 9 later cherry-picks), `29fc8f1` twice (Task 11's +return contract, Task 14's caller guard), and `21869e2 + 29fc8f1` / +`e116feb + 29fc8f1` / `0a57e41 + 10d598f` as pairs. + +- [ ] **Step 8: Record the migration state the deploy needs** + +```bash +.venv-test/bin/python -m alembic heads +.venv-test/bin/python -c " +import sys; sys.path.insert(0, 'scripts/migrate') +import preflight as pf +print('preflight target:', pf.DEFAULT_TARGET) +print('supported starts:', pf.SUPPORTED_START_REVISIONS) +print('org1 is at 0018 — the expensive path. See docs/production-migration.md §3.') +" +``` + +Expected: head `0024`, target `0024`, and `0018` present in the supported starts. + +- [ ] **Step 9: Stop. Do not deploy.** + +The branch is code-complete and gate-green. Deployment is a **separate, planned outage window** — `0018 -> 0024`, with `ACCESS EXCLUSIVE` on `agent_messages` and a hard failure if duplicate `(simulation_run_id, message_ts)` rows exist. Follow `docs/production-migration.md`: measure (§2), remediate duplicates, rehearse with `preflight.py`, then +`COMPOSE_FILE=docker-compose.prod.yml ./scripts/migrate/run_migration.sh --apply`, then `postflight.py`. Migrate **before** rebuilding and restarting, and rebuild the `agent` image separately (`--profile agent build agent`) because it bakes `src/`. + +Note the rollback window: `0019`'s downgrade sets `agent_messages.agent_id` back to `NOT NULL`, so it works immediately after the window but fails as soon as the new code writes its first `agent_id=NULL` row. After that the only rollback is the dump `run_migration.sh` takes at step 4. + +--- + +## Follow-up, deliberately not in this plan + +These are recorded so they are not mistaken for oversights. None blocks the branch. + +1. **org1's `CLAUDE.md` needs authoring** (spec §8). This plan drops blackbird's + CLAUDE.md hunks (Task 9 Step 1) and does not write a replacement. Two things are + worth salvaging from blackbird's version, with every `blackbird-app` / + `blackbird-agent-run` / two-stack reference discarded: its **Testing** section, which + is a doc-accuracy fix for both instances (cohort's `CLAUDE.md` still describes the + in-container pytest path and omits the alembic round trip and `src/` ratchet that + cohort's own `cc8490f` added), and the "nothing migrates the database for you" + warning, which is true of org1's bare-`uvicorn` prod command too. +2. **Whether to enable post-type enforcement when cohorts are turned on**, and whether + that change authors an org1 prompt variant or keeps prompts frozen (spec §7.1). +3. **How long to hold the `0019` rollback door open** before the first `agent_id=NULL` + row makes `alembic downgrade` unusable (spec §5). +4. **`copi-prod`'s future.** Once this branch reaches `main`, `copi-prod` should be + deleted or reduced to a deploy tag, so a fifth divergent line does not re-accumulate + (spec §9). +5. **`coPI-podcast`** carries 66 unmerged commits (podcast/TTS, PI proposal evaluations, + focus-agent mode). A separate reconciliation, untouched here. diff --git a/docs/production-migration.md b/docs/production-migration.md new file mode 100644 index 0000000..b8d8a77 --- /dev/null +++ b/docs/production-migration.md @@ -0,0 +1,597 @@ +# Production migration to alembic 0024 (`org1-parity`) + +**Audience: an operator or agent who has not done any of the analysis behind this.** +You do not need to understand the branch to run this. You do need to follow the order, +and you need to stop when something says STOP. + +Supported starting points: **0018** (`main` before PR19), **0019**, **0020** and **0021**. +All four are tested end to end. + +**If your deployment tracks `main`, you are at 0021** — that is `origin/main`'s own alembic +head, since PR19 merged 0019, 0020 and 0021. Starting there is the *easiest* case: the +expensive 0019 index build and the duplicate-row risk are both already behind you, and all +that remains is 0022 (three empty tables) and 0023 (three columns on a small table), which +takes ~2 s at any size. Starting from 0018 is the one that needs a real window. + +The executable half of this runbook is `scripts/migrate/run_migration.sh`. This document +explains *why* each step is where it is, which is what you need when a step fails. + +--- + +## 0. The five hard rules + +1. **Take the backup. Verify the backup.** Migration 0019 is a one-way door (§9). + `run_migration.sh --apply` does both for you and refuses to continue if either fails. +2. **Migrate the database BEFORE deploying the application code.** Not the other way + round. §8 explains what breaks in each direction. +3. **`alembic downgrade` is not a rollback.** It either destroys data silently or refuses + to run. Your rollback is a restore from the dump. See §9 — read it before you start, + not after. +4. **Never let the DSN default, and always run these tools inside the container.** + `alembic.ini` falls back to `postgresql+asyncpg://copi:copi@localhost:5432/copi`, and + `env.py` only overrides it when `DATABASE_URL` is set — so a migration run with no DSN + targets whatever answers on `localhost:5432`. Two measured facts on this machine: + the compose file does **not** publish Postgres to the host (nothing listens on + 127.0.0.1:5432, so a host-side run fails closed), but the bare hostname `postgres` + resolves from the host to **195.35.25.84 — a public IP** via a LAN search domain. A DSN + copied out of this runbook and run on the *host* therefore points at a stranger's + server, not your database. Inside the container `postgres` is the compose service and is + correct. Always pass `--database-url` or export `DATABASE_URL`; + `run_migration.sh` refuses to run without one and does the `docker compose exec` for you. +5. **Alembic's own output is not evidence.** "Running upgrade 0018 -> 0019" is printed + before the transaction commits. A bad `env.py` once made all 18 migrations log success + and then silently roll the entire chain back, leaving no `alembic_version` row at all. + Always read the revision back out of the database. Step 6 of the script does this. + +--- + +## 1. What this migration actually does + +Five revisions, applied as one chain: + +| Revision | Change | +|---|---| +| `0018 -> 0019` | 7 new columns on `agent_messages` (the DB becomes the primary message store), 3 new indexes, the unique constraint `uq_agent_messages_run_ts` (itself backed by a 4th index), and `agent_id` becomes nullable. This is the expensive one. | +| `0019 -> 0020` | Creates `pi_dm_messages` (+2 indexes) and the `pi_dm_direction_enum` type. | +| `0020 -> 0021` | 2 indexes for the DB inbox pollers' `created_at` cursor. | +| `0021 -> 0022` | Creates `cohorts`, `cohort_memberships`, `cohort_audit_events` (+4 indexes). | +| `0022 -> 0023` | 3 synthesis-provenance columns on `researcher_profiles`. | +| `0023 -> 0024` | One `VARCHAR(20) NOT NULL DEFAULT 'pi_lab'` column on `agents`. Postgres 11+ fills a non-volatile default without a table rewrite, and `agents` is small, so this is seconds at any size. | + +Verified properties of the chain, measured rather than assumed: + +- **No migration in the chain issues a data-mutating statement against existing rows.** + Counted directly from the five migration files: 0 `op.execute`, and 0 `UPDATE` / + `INSERT INTO` / `DELETE FROM` in any `upgrade()`. Confirmed against live data too — an md5 + fingerprint over all 11 pre-0019 columns of `agent_messages` is byte-identical at every + revision from 0018 to 0023. +- **No table rewrite.** `pg_class.relfilenode` for `agent_messages` is unchanged across the + chain. The new columns are added with non-volatile defaults, which Postgres 11+ applies + as metadata only. +- **Partial application is impossible.** `alembic/env.py` deliberately does *not* pass + `transaction_per_migration`, so all five revisions run in a single transaction. Verified + by `pg_terminate_backend`-ing the backend mid-chain twice: both times the database came + back at the original revision with nothing applied. +- **Existing rows keep their row count.** Confirmed on a 463-row production-like fixture: + 463 before, 463 after. + +The cost is that every lock the chain takes is held until the final commit, and 0019 takes +`ACCESS EXCLUSIVE` on `agent_messages`. That is why §3 and §4 exist. + +--- + +## 2. Measure production first (read-only, safe to run any time) + +Run these against production **before** you plan the window. They are pure `SELECT`s. +They work at 0018 and at 0023, so you can also run them afterwards to compare. + +Open a psql shell in the container (no `-T` — you want a terminal): + +```bash +docker compose exec postgres psql -U copi -d copi +``` + +```sql +-- Q1. Scale. Drives how long the lock is held (§3). +SELECT (SELECT count(*) FROM agent_messages) AS agent_messages_rows, + pg_size_pretty(pg_table_size('agent_messages')) AS heap, + pg_size_pretty(pg_indexes_size('agent_messages')) AS indexes, + pg_size_pretty(pg_total_relation_size('agent_messages')) AS total, + (SELECT count(*) FROM simulation_runs) AS runs; + +-- Q2. Will migration 0019 abort? Anything other than 0 means STOP and read §5. +SELECT count(*) AS duplicate_groups, + coalesce(sum(n) - count(*), 0) AS rows_above_one_per_group +FROM (SELECT simulation_run_id, message_ts, count(*) AS n + FROM agent_messages WHERE message_ts IS NOT NULL + GROUP BY 1, 2 HAVING count(*) > 1) d; + +-- Q3. Legacy inventory and rollback blockers (§9). +SELECT count(*) AS total, + count(*) FILTER (WHERE agent_id IS NULL) AS blocks_downgrade_past_0019, + count(*) FILTER (WHERE message_ts IS NULL) AS null_message_ts, + count(*) FILTER (WHERE message_ts LIKE 'local:%') AS locally_minted, + count(*) FILTER (WHERE message_ts IS NOT NULL + AND message_ts NOT LIKE 'local:%') AS slack_shaped +FROM agent_messages; + +-- Q4. Anything that would block (or be blocked by) the ACCESS EXCLUSIVE lock. +-- An `idle in transaction` row here is the dangerous one: it will never finish on its own. +SELECT pid, state, now() - xact_start AS xact_age, left(query, 60) AS query +FROM pg_stat_activity +WHERE datname = current_database() AND pid <> pg_backend_pid() + AND xact_start IS NOT NULL +ORDER BY xact_start LIMIT 10; +``` + +Also check free disk, because 0019 and 0021 add six indexes to `agent_messages` (three plus +the unique constraint's, then two more). Ask the container about its +own data directory rather than guessing the volume name on the host: + +```bash +docker compose exec -T postgres df -h /var/lib/postgresql/data +docker compose exec -T postgres psql -U copi -d copi \ + -c "select pg_size_pretty(pg_database_size(current_database()))" +``` + +If `Use%` is in the high 90s, stop and reclaim space first. A full data volume during index +creation is a much worse failure than a postponed window. (`docker system prune` and +`docker builder prune` are usually where the space went on a dev box; do not run either +against a production host without knowing what is on it.) + +Preflight (§6) runs stricter versions of all of these and blocks on them. Q1–Q4 exist so +you can size the window *before* touching anything. + +--- + +## 3. How long the window needs to be + +Measured on seeded copies of this schema, on the machine this tooling was built on. Treat +them as order-of-magnitude for your own hardware, and re-measure on a restored copy if the +window is tight: + +| `agent_messages` rows | `0018 -> 0021` | `0021 -> 0023` | +|---|---|---| +| 10,000 | ~0.11 s | ~2 s | +| 100,000 | ~0.75 s | ~2 s | +| 500,000 | ~5.1 s | ~2 s | +| 1,000,000 | ~7.9 s | ~2 s | +| 2,000,000 | ~30.6 s | ~2 s | + +The second hop is effectively constant — it creates empty tables and adds columns to a +small table. All the cost is index-building in 0019/0021, which scales with row count. +Preflight check 9 makes the same estimate from your actual row count and prints it, so you +do not have to interpolate this table by hand. + +Index storage for `agent_messages` grew from **96 MB to 565 MB at 2.5 M rows** in testing. +Size your headroom from Q1, not from that number. + +**Writes to `agent_messages` are blocked for the whole window.** This was verified, not +inferred: a concurrent writer blocks until the chain commits. Reads that start *before* +the migration continue; reads that arrive *after* the `ACCESS EXCLUSIVE` request queues +behind it and also block. Treat the window as a full outage on that table. + +--- + +## 4. The lock timeout, and why you want it + +`ALEMBIC_LOCK_TIMEOUT_MS` defaults to **10000** (10 s). It bounds only how long the +migration *waits to acquire* a lock. It is not `statement_timeout` and will not cancel a +legitimately long index build partway through. + +Without it, one forgotten `BEGIN; SELECT …` parks the migration forever, and because a +pending `ACCESS EXCLUSIVE` request queues ahead of new readers, every subsequent query on +`agent_messages` stalls behind it — an unbounded outage with nothing to end it. + +Verified behaviour with a real blocker holding `AccessShareLock`: the migration failed +after ~12 s with `LockNotAvailableError`, the transaction rolled back cleanly, and +`alembic_version` was still `0018`. **A lock timeout costs you nothing but the attempt.** + +If you hit it: stop the writers and re-run. + +```bash +docker stop -t 30 agent-run # SIGTERM; -t 30 lets an in-flight LLM call finish +``` + +Do **not** use `docker rm -f` / `kill -9` on `agent-run`: SIGKILL skips the shutdown flush +and permanently loses the in-flight turn's messages. The DB, not Slack, is the durable +store. + +It is an environment variable, not a flag. Raise it only if you have a specific reason: + +```bash +ALEMBIC_LOCK_TIMEOUT_MS=30000 ./scripts/migrate/run_migration.sh --apply +``` + +`0` means wait forever. Don't. + +--- + +## 5. Duplicate `(simulation_run_id, message_ts)` rows + +Migration 0019 creates `uq_agent_messages_run_ts`. If duplicates exist, the migration +aborts — and **Postgres names only ONE conflicting key per failed index build**, so +fixing them by reading the error message means one migration attempt per duplicate group. + +Preflight check 4 lists **all** groups with their row ids in one pass (up to +`--max-duplicate-groups`, default 200; the *count* is never truncated, and the output tells +you when it has truncated the listing). + +`NULL` `message_ts` rows are excluded on purpose: Postgres `UNIQUE` treats NULLs as +distinct, so they cannot violate the constraint. Verified — three NULL-ts rows in one run +coexist with the constraint. + +### Fixing them + +```bash +DSN=postgresql+asyncpg://copi:copi@postgres:5432/copi + +# Dry run. Runs in a READ ONLY transaction — it cannot write. Verified inert by +# checksumming the table before and after. +docker compose exec -T -e PYTHONPATH=/app -e DATABASE_URL="$DSN" app \ + python scripts/migrate/remediate_duplicates.py + +# Apply. Takes SHARE ROW EXCLUSIVE on agent_messages, re-checks inside the same +# transaction, and rolls back if any group would remain. +docker compose exec -T -e PYTHONPATH=/app -e DATABASE_URL="$DSN" app \ + python scripts/migrate/remediate_duplicates.py --apply +``` + +Strategies: + +- **`renumber`** (default, non-destructive): never deletes a row. Gives duplicates new + locally-minted ids where that is safe. **Use this.** +- `keep-earliest` / `keep-latest` (destructive, opt-in): additionally `DELETE` the + redundant copies of byte-identical groups. + +Divergent groups — two rows sharing a key but with *different* content — are renumbered, +never deleted, under every strategy. Two rows that both carry a real Slack timestamp and +disagree are refused entirely and reported as `needs_human`: that combination means +something upstream is wrong and a script guessing which one is canonical would be worse +than stopping. + +Exit codes: `0` clean/applied · `1` duplicates remain or would remain · `2` found in a dry +run, all resolvable · `3` operational failure · `64` usage error. (`64`, not argparse's +`2`, because `2` already means "duplicates found".) + +--- + +## 6. Run the migration + +### 6a. Rehearse. This writes nothing. + +```bash +export DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi +./scripts/migrate/run_migration.sh +``` + +Every tool in `scripts/migrate/` is dry-run by default, deliberately: an operator who +learns the convention from one must not be caught out by another. + +Exit `0` = clear · `1` = **STOP**, a check failed · `2` = warnings, your judgement · +`3` = operational failure · `64` = usage error. + +Exit 2 is the normal outcome of a first rehearsal on real data: checks 11 and 12 warn (see +below). It means "read these, then decide", not "something is broken". In `--apply` mode +warnings do not stop the run, because choosing to apply *is* the decision — so a successful +apply is `0` even if preflight warned. + +The 13 preflight checks: + +``` + 1. Stamped alembic revision is a supported starting point + 2. Exactly one alembic head, no duplicate revision ids + 3. The 0019 stamp is the content 0019, not one of the other 0019s + 4. No duplicate (simulation_run_id, message_ts) in agent_messages + 5. Objects the pending revisions create do not already exist + 6. Rows that would block a downgrade past 0019 (agent_messages.agent_id IS NULL) + 7. No sessions that would block (or be blocked by) the ACCESS EXCLUSIVE lock + 8. Migration harness commits what it applies (alembic/env.py) + 9. Sizing and expected lock window +10. Disk headroom for the indexes 0019/0021 add +11. Legacy-row inventory (rows that will have content = '') +12. Recent, non-trivial backup exists +13. Row-count snapshot written for postflight +``` + +Check 3 deserves a note. **Three** different files in this repository's history declared +`revision = "0019"`, all revising 0018 — enumerated by parsing every historical blob under +`alembic/versions/`, not from memory: + +| File | Branch | Signature it leaves | +|---|---|---| +| `0019_agent_message_content.py` | this chain — the one you want | `agent_messages.content` | +| `0019_add_cohorts.py` | `cohort-agent-isolation` | `cohorts` table | +| `0019_add_hidden_to_proposals.py` | `coPI-podcast` | `thread_decisions.hidden` | + +A database stamped `0019` by either of the other two is missing the content columns the +application requires, and **`alembic upgrade` does not notice**. Both outcomes were measured +on fixtures in exactly those states: + +- **Cohort 0019** — `alembic upgrade 0023` applies 0020 and 0021, then dies at 0022 with + `DuplicateTableError: relation "cohorts" already exists`. The revision stays `0019` and + nothing is applied (one transaction). Loud, and safe. +- **Podcast 0019** — `alembic upgrade 0023` **exits 0 and stamps `0023`**, having run 0020, + 0021, 0022 and 0023 without complaint, while `agent_messages.content` does not exist and + neither does `uq_agent_messages_run_ts`. Alembic reports complete success on a database + the application cannot run against. This is the silent one, and it is why check 3 exists. + +Check 3 probes for each signature separately and names the one it actually found, because +the remediation differs: the cohort tables must be dropped so 0022 can create them properly, +whereas the two `hidden` columns are orphaned but harmless and are better left in place. If +it finds a `0019` stamp matching none of the three, it refuses and tells you to inspect by +hand rather than guess. + +If preflight is somehow bypassed, **postflight is the backstop**: run against the silently +"successful" podcast-0019 database it reports **6 FAIL, exit 1**. That is the whole reason +step 7 checks the schema instead of trusting the revision stamp. + +Checks 11 and 12 normally `WARN`. Check 11 warns because legacy rows genuinely will have +`content = ''` (§7). Check 12 warns in rehearsal mode because no dump was taken. Read +both; neither blocks. + +### 6b. Apply. + +```bash +./scripts/migrate/run_migration.sh --apply +``` + +Seven steps, in this order and for these reasons: + +1. **Container runs current code.** The `Dockerfile` does `pip install .`, baking a copy of + `src/` into site-packages. For `python scripts/X.py`, CPython sets `sys.path[0]` to the + script's directory, so `/app` is *not* on the path and `import src` resolves to the + baked — possibly days-old — copy. Every step passes `PYTHONPATH=/app`; this step proves + it worked by asserting `src.__file__ == /app/src/__init__.py` and that `Cohort` imports. +2. **Resolve the DSN and print it** (password masked). Refuses to run without one. +3. **Backup**, before preflight, so a *blocked* preflight still leaves you with a dump. + Dumps `-Fc` inside the container, verifies the archive is readable with `pg_restore -l` + there, then copies it to the host and re-checks the size. A dump whose table of contents + cannot be read is a file, not a backup. +4. **Preflight.** Exit 1 stops here. +5. **`alembic upgrade`** — one command, so the whole chain is one transaction. +6. **Read `alembic_version` back out of the database.** See rule 5 in §0. +7. **Postflight.** + +If you have a verified backup the script cannot see (managed snapshots, base backup + WAL): + +```bash +./scripts/migrate/run_migration.sh --apply \ + --backup-verified-elsewhere "nightly base backup + WAL, restore tested 2026-08-04" +``` + +That flag makes you *write down* what you are asserting, and the reason is echoed into the +run's output. It is not a way to skip having a backup, and it is the **only** way to stop +this script taking its own dump — there is deliberately no bare "skip the backup check" +flag, so nobody can turn the check off without stating a reason. + +### 6c. What postflight proves + +13 checks. Check 1 is the revision stamp, and its own output says the stamp proves nothing +on its own — the other 12 check the schema: + +``` + 1. alembic_version is exactly the target revision + 2. Exactly one alembic head, no duplicate revision ids + 3. Every table 0020/0022 creates exists + 4. Every column 0019/0020/0023 adds exists, with the right type and nullability + 5. Every index 0019/0020/0021/0022 creates exists, on the right columns + 6. Constraints 0019/0022 add exist with the right definition + 7. pi_dm_direction_enum has exactly the expected values + 8. No invalid indexes (pg_index.indisvalid / indisready / indislive) + 9. No unintended NULLs in the columns the migrations declare NOT NULL +10. No foreign-key orphans, and every FK is convalidated +11. Row counts match the preflight snapshot +12. No ORM drift (nothing the models require is absent from the database) +13. The ORM at HEAD can query every model +``` + +Check 11 compares against the snapshot preflight wrote, which is why the two must be run +as a pair — `run_migration.sh` handles that. Checks 12 and 13 are the ones that catch +"schema applied but the application still can't run". + +**Postflight must be 0 FAIL before you deploy code.** + +--- + +## 7. Legacy rows: what the migration cannot give back + +0019 adds `content` with a default of `''` and `posted_at` with a default of `0`. Rows that +existed before the migration therefore end up claiming *"this message had an empty body and +was posted at the Unix epoch"*. That is a semantically false statement about real data, and +no migration can fix it, because the bodies were never in the database — they were only +ever in Slack. + +Consequences you should expect, and which are already handled in the code: + +- Legacy rows all share `posted_at = 0`. Any `ORDER BY posted_at DESC … LIMIT n` therefore + has ties, and Postgres may return a *different* page each time. `src/routers/agent_page.py` + orders by `posted_at DESC, created_at DESC, id DESC` — a total ordering — for exactly this + reason. If you add a paged query over `agent_messages`, do the same. +- Preflight check 11 splits legacy rows into **Slack-recoverable** and **permanently + unrecoverable**. Read that number before the window so nobody is surprised by it after. + +Step 8 recovers what Slack still has. + +--- + +## 8. After the migration, in this order + +### Step 8 — repair the Slack mirror mapping + +```bash +docker compose exec -T -e PYTHONPATH=/app app python scripts/backfill_slack_ts.py # report +docker compose exec -T -e PYTHONPATH=/app app python scripts/backfill_slack_ts.py --apply # write +``` + +This asks Slack which timestamps actually exist and writes only confirmed ones. Rows Slack +does not recognise are left `NULL`, which is now the truthful value — the code no longer +*infers* the mapping, because inferring fabricated timestamps that were then handed to +`chat.postMessage` as a `thread_ts`. + +It needs a valid bot token in every affected channel. It is read-only against Slack, only +ever writes `slack_ts`, and is safe to re-run. + +**Exit 2 means some rows were UNVERIFIED — not that they were absent.** Unverified means +Slack did not answer for them (rate limit, token missing from that channel, channel +archived). Re-run once the cause is fixed. Do not read exit 2 as "done". + +### Step 9 — deploy the application code, then restart + +```bash +docker compose up -d --build app worker +``` + +**Order matters, and only one order is safe.** The new code requires columns that only +exist at 0023, so code-before-migration fails immediately and obviously. Migration-before-code +is the safe direction: the old code does not reference the new columns, and the new columns +all have defaults, so old code keeps working against the new schema during the gap. + +One real gap exists in that window: `_slack_parent_ts_from_db` has no content filter, so a +private-channel close marker can resolve its parent to `None` and not be mirrored to Slack. +Messages themselves keep mirroring correctly — `_slack_parent_ts` returns `thread_ts` when +the root row is missing. (PR19's own deploy-order warning claims replies stop being +mirrored. That claim is wrong; this is what actually breaks.) Keep the gap short and this +costs you one marker. + +### Step 10 — start the simulation last + +```bash +docker compose --profile agent run -d --name agent-run agent python -m src.agent.main --budget 0 +``` + +Last, because it is the heaviest writer to `agent_messages`. Starting it before app+worker +are up on the new code means it writes through code paths the rest of the deployment does +not yet agree with. + +Roster changes do **not** need a restart (`_sync_roster_from_db` re-reads every ~30 s), but +**code** changes do: the process only loads modules at startup. + +--- + +## 9. Rollback: read this before you start + +### `alembic downgrade` is not a rollback. Both of its outcomes are bad. + +Verified on live databases, twice, just now: + +**If no PI messages exist** (`agent_id IS NULL` count is 0 in Q3): + +``` +$ alembic downgrade 0018 +exit=0 +rows=463 <- unchanged +content column: GONE +pi_dm_messages: GONE +cohorts: GONE +``` + +It **exits 0**, reports success, preserves the row count exactly — and destroys every +message body, every PI direct message, and every cohort. A row-count check will not notice. +This is the single most dangerous command in this runbook. + +**If any PI message exists:** + +``` +$ alembic downgrade 0018 +sqlalchemy.exc.IntegrityError: NotNullViolationError: + column "agent_id" of relation "agent_messages" contains null values + [SQL: ALTER TABLE agent_messages ALTER COLUMN agent_id SET NOT NULL] +exit=1 +revision now: 0023 <- unchanged, and the PI row survived +``` + +It refuses. The single transaction rolls back cleanly and nothing is lost — but you have no +downgrade path. Note the shape of this: the downgrade is blocked *exactly when* there is +real PI data to protect, and succeeds destructively *exactly when* the bodies it deletes are +the only copy. + +**Therefore: your rollback is a restore from the dump.** + +### If postflight fails + +Do **not** deploy application code. Nothing is half-applied — the chain is one transaction, +so either it all committed or none of it did. Postflight failing after a committed chain +means the schema is not what 0023 should produce, which is a bug to investigate, not a +partial state to repair. + +1. Read which checks failed. Checks 3–7 name the exact missing object. +2. Confirm the revision independently: + ```bash + docker compose exec -T postgres psql -U copi -d copi -c 'select * from alembic_version' + ``` +3. If you need to get back to where you started, restore the dump: + ```bash + docker stop -t 30 agent-run || true + docker compose stop app worker + + docker compose cp backups/copi_pre0023_.dump postgres:/tmp/restore.dump + docker compose exec -T postgres psql -U copi -d postgres \ + -c 'ALTER DATABASE copi RENAME TO copi_failed_migration' + docker compose exec -T postgres psql -U copi -d postgres -c 'CREATE DATABASE copi' + docker compose exec -T postgres pg_restore -U copi -d copi --exit-on-error /tmp/restore.dump + + docker compose exec -T postgres psql -U copi -d copi -c 'select * from alembic_version' + ``` + Rename rather than drop: keep the failed database until you have confirmed the restore + is good. `--exit-on-error` is not optional — without it `pg_restore` reports success + after partially restoring. +4. Verify the restore before starting anything: row counts against Q1/Q3, and the revision + should read `0018` or `0019` again. + +--- + +## 10. Quick reference + +| | | +|---|---| +| Orchestrator | `scripts/migrate/run_migration.sh` — `0` clear/applied · `1` blocked · `2` rehearsal raised warnings · `3` operational · `64` usage | +| Preflight | `scripts/migrate/preflight.py` — `0` ok · `1` blocked · `2` warnings | +| Postflight | `scripts/migrate/postflight.py` — `0` verified · non-zero: do not deploy | +| Duplicates | `scripts/migrate/remediate_duplicates.py` — `0` clean · `1` remain · `2` found (dry run) · `3` operational · `64` usage | +| Slack mapping | `scripts/backfill_slack_ts.py` — `0` all verified · `2` some UNVERIFIED | +| Lock wait | `ALEMBIC_LOCK_TIMEOUT_MS`, default `10000` ms | +| Backup dir | `MIGRATE_BACKUP_DIR`, default `backups/` (gitignored) | +| Services | `MIGRATE_SERVICE` (default `app`), `MIGRATE_PG_SERVICE` (default `postgres`) | + +Every Python tool here takes `--database-url` and defaults to `$DATABASE_URL`; all are +dry-run unless given `--apply`; all must be run with `PYTHONPATH=/app` inside the container. + +--- + +## 11. What has been tested, and what has not + +Tested end to end on seeded production-like databases: + +- **From 0018**, with 463 rows (300 Slack-born, 120 `local:`, 40 NULL-`message_ts`) and 3 + planted duplicate groups: rehearsal blocked with all 3 groups listed → remediation dry + run inert (checksum identical) → `--apply` cleared them → migration applied → revision + read back as 0023 → postflight 13 checks, 0 FAIL → 463 rows preserved. +- **From 0019**, with 151 rows including a PI row (`agent_id IS NULL`): preflight warned + (correctly) that downgrade is blocked, migration applied, revision 0023, postflight 13 + checks 0 FAIL, 151 rows preserved. +- **From 0020 and 0021**, with 120 rows each: check 1 passes, check 9 correctly reports + that 0019's index build is already behind you rather than quoting a row-scaled window, + migration applied, revision 0023, postflight 13 checks 0 FAIL, 120 rows preserved. +- Lock timeout against a real blocker: failed fast at ~12 s, revision unchanged. +- Both downgrade outcomes in §9, on live databases. +- Mid-chain `pg_terminate_backend`, twice: no partial application. +- **The restore path in §9, as a full drill.** Dump a seeded 0018 database → migrate to + 0023 → destroy half the rows → run the §9 commands verbatim → 463 rows back, revision + back to `0018`, and an md5 over `(id, message_ts, agent_id, channel_id)` for all 463 rows + **byte-identical to the pre-migration source**. Then re-ran the whole migration on the + restored database: 0023, postflight 0 FAIL, 463 rows. So the dump restores, and what it + restores can be migrated again — you get a second attempt, not just your data back. + +**Not tested, and you should know it:** + +- Any database larger than ~2.5 M `agent_messages` rows. The §3 timings are extrapolation + beyond that. +- The restore drill above was run against a scratch database in this same cluster, with the + `copi` role and extensions already present. It did not include starting the application + against the restored database. **Do the drill on a copy of your own production data before + your window**, not during it. +- A managed/hosted Postgres. Everything here assumes the `postgres` compose service and + `docker compose exec`. On RDS or similar, the SQL and the alembic steps carry over; the + backup and `docker compose exec` plumbing does not. +- Replication. Nobody checked what a standby does with a 30-second `ACCESS EXCLUSIVE` hold. diff --git a/docs/specs/2026-08-05-conversations-cohort-scope-and-threads-design.md b/docs/specs/2026-08-05-conversations-cohort-scope-and-threads-design.md new file mode 100644 index 0000000..a556c0f --- /dev/null +++ b/docs/specs/2026-08-05-conversations-cohort-scope-and-threads-design.md @@ -0,0 +1,378 @@ +# Design — cohort-scoped conversations feed, threaded display, and topology payload restructure + +**Status:** DESIGN, not implemented. +**Date:** 2026-08-05 +**Target branch:** `blackbird`. +**Companions:** `specs/cohort-system-v2.md` (gate semantics, §5/§6/§12), +`specs/privacy-and-channel-visibility.md` (visibility classes), +`specs/local-db-conversations.md` (DB as the primary conversation store). + +--- + +## 1. Problem + +Three defects, two of them in the same page. + +### 1.1 The conversations feed is not cohort-scoped + +`GET /agent/{agent_id}/conversations` (`src/routers/agent_page.py:702`) selects +messages with **channel name as the only content filter** +(`agent_page.py:755-772`): + +```python +select(AgentMessage).where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.channel_name.in_(channels), +) +``` + +There is no `agent_id` filter, no `visibility` filter, and no cohort gate. The +channel set (`agent_page.py:747-753`) is "channels this agent authored in" unioned +with `{"general"}` unconditionally, so **every PI sees every other lab's bot traffic +in `#general`**, and sees all bot traffic in any channel their own bot has posted +in. + +This directly contradicts the deployed topology. Runtime settings are +`cohort_isolation_enabled=True` and `cohort_default_policy="isolated"` (the +`src/config.py:330` default of `False` is overridden in this deployment), and the +membership table is a **star**: 56 agents in exactly one cohort each, 2 agents in +all 56. The engine therefore already prevents a spoke bot from *acting on* another +spoke's posts — but the PI-facing page shows them anyway. The page shows strictly +more than the bot it represents is allowed to see. + +The dashboard, by contrast, does scope correctly: `AgentMessage.agent_id == aid` at +`agent_page.py:198-212`. + +### 1.2 Threads are not viewable + +The feed is a flat list of the 100 newest messages. `AgentMessage` carries +everything needed to thread them — `message_ts` (canonical id, unique per run) and +`thread_ts` (NULL on roots, equal to the root's `message_ts` on replies), plus +`phase ∈ {new_post, thread_reply}` (`src/models/agent_activity.py:88-93`) — but the +row dict built at `agent_page.py:774-784` passes `thread_ts` and **omits +`message_ts`**, so a reply has no root to attach to. The template can only render a +`· thread` badge (`templates/agent/conversations.html:82`). A PI cannot read +responses to their bot's posts. + +### 1.3 The topology matrix cannot be saved + +`POST /admin/cohorts/topology` fails with +`Too many fields. Maximum number of fields is 1000.` + +`templates/admin/cohort_topology.html:82` emits one hidden `present` input **per +rendered cell**. At 60 agents × 56 cohorts that is 3,360 hidden fields plus 168 +ticked checkboxes ≈ **3,528 form fields**. `admin.py:1559` calls +`await request.form()` with no arguments, and Starlette 1.4.1 defaults +`max_fields=1000`, raising at `starlette/formparsers.py:96`. FastAPI surfaces it as +a 400 `detail`. The page renders fine; only the POST fails. It broke silently once +`agents × cohorts` crossed ~1,000 cells. This is the only `request.form()` call in +the codebase. + +## 2. Requirements (settled during brainstorming) + +- **Gate rule:** the page mirrors the engine's `_entry_allowed` + (`src/agent/message_log.py:48-80`) **exactly**, including both documented + bypasses — humans always pass, `collab_private` always passes. One narrow, + deliberate exception ships on top of this mirror — see §4.3. +- **Thread fetch:** roots only on first paint, replies loaded **on click** from a + new endpoint. +- **Thread gating:** replies **are** gated, and the reply count is computed with the + same gate so the badge never promises turns the expansion will not show. This is a + deliberate divergence from the engine, which classifies `get_thread_history` as + UNGATED. +- **Topology:** restructure the payload to 116 markers; do not weaken the diff + safety property. + +### 2.1 Explicitly out of scope + +The **private-channel read leak** is real but latent and is *not* fixed here. The +write path is gated by `pi_may_post_to_channel` (`src/services/pi_inbox.py:52-100`, +which honours `removed_at`); the read path has no equivalent, so a PI can read a +`collab_private` channel's history from before their bot joined, and read access +survives membership revocation. There are currently **zero** `collab_private` +channels in the database. Mirroring `_entry_allowed` (requirement above) means +`collab_private` keeps its blanket pass. Track separately. + +## 3. Approach + +The gate becomes one set of semantics expressed in two places that are provably in +sync. `_entry_allowed` remains the in-memory predicate for the engine; a new +`gate_clause()` renders the same truth table as a SQLAlchemy `WHERE` fragment for +the web page. `src/services/cohorts.py:1-8` already establishes this pattern — "The +simulation engine applies the gate; the admin UI previews it. They must never +disagree" — so the new module follows that precedent instead of inventing a parallel +one, and §7 pins the equivalence with a parity test. + +Three deliverables: + +| # | Change | Files | +|---|---|---| +| A | Cohort-scoped visibility | new `src/services/conversation_feed.py`, `src/routers/agent_page.py` | +| B | Threaded display with lazy expand | `agent_page.py` (+1 route), `conversations.html`, new `_thread_replies.html` | +| C | Topology payload 3,360 → 116 | `templates/admin/cohort_topology.html`, `src/routers/admin.py` | + +C is independent and lands first. A and B share the gate primitive, so A precedes B. + +## 4. Component: `src/services/conversation_feed.py` (new) + +Owns one question: *what may this PI see in the conversation feed, and how do I ask +Postgres for it.* + +```python +async def resolve_agent_gate(db, agent_id: str) -> set[str] | None +def gate_clause(gate: set[str] | None) -> ColumnElement[bool] +``` + +### 4.1 `resolve_agent_gate` + +Calls `compute_gates` (`src/services/cohorts.py:74`) exactly as +`_cohort_gate_context` does (`src/routers/admin.py:1372-1401`), with **one +deliberate difference**: the roster is `active agents ∪ {viewing agent}`. + +The route admits `status in ("active", "inactive")` (`agent_page.py:718`), but +`compute_gates` only returns keys for the roster it is given, so an inactive viewing +agent would raise `KeyError`. Including it can only *lower* the chance of a +preflight refusal — it raises `live_members`, which the refusal test at +`cohorts.py:59-70` compares against zero — so it cannot cause a silent +roster-wide-silence regression. + +Returns the viewing agent's `allowed_sender_ids`: `None` (gate off), or a set +(possibly empty). + +### 4.2 `gate_clause` + +The SQL mirror of `_entry_allowed`, ordered clause for clause to make the +correspondence reviewable: + +```python +if gate is None: # gate off for this agent + return true() +return or_( + AgentMessage.is_bot.is_(False), # humans pass + AgentMessage.visibility == VISIBILITY_COLLAB_PRIVATE, # explicit pairing passes + and_(AgentMessage.agent_id.is_not(None), # fail closed on NULL + AgentMessage.agent_id.in_(gate)) if gate else false(), +) +``` + +The `if gate else false()` is load-bearing: an uncohorted agent under +`policy="isolated"` gets `set()`, and an empty `IN` is the one input whose rendering +would otherwise be ambiguous. + +`VISIBILITY_COLLAB_PRIVATE` is imported from `src/visibility.py:18` — the same +constant `_entry_allowed` uses, not a string literal, so the two cannot drift apart +on a rename. + +The `is_bot` keying (rather than `agent_id is None`) and the NULL-`agent_id` +fail-closed branch are both carried over from `_entry_allowed`'s docstring, which +records why each exists: `agent_messages.agent_id` is nullable, so a bot-authored +row with a NULL `agent_id` would otherwise pass through the human bypass. + +### 4.3 `own_or_gated` — the one deliberate divergence from `_entry_allowed` + +Shipped alongside `gate_clause` in `src/services/conversation_feed.py` but not +enumerated in §4.2's mirror above (an audit gap — this subsection is the fix): + +```python +def own_or_gated(gate: set[str] | None, agent_id: str) -> ColumnElement[bool]: + return or_(gate_clause(gate), AgentMessage.agent_id == agent_id) +``` + +`gate_clause` alone is not quite what every call site needs. Under +`policy="isolated"`, an agent that is active but not yet placed in any cohort +gets `gate == set()` (§4.1, `resolve_agent_gate`/`compute_gates`), and +`gate_clause(set())` — correctly, per `_entry_allowed` — admits nothing from the +membership branch. That is exactly right for the *engine*: an uncohorted agent +should not act on anyone. It is wrong for the *PI's own page*: onboarding +activates an agent before an admin has assigned it to a cohort (see +`CLAUDE.md`'s Provision → Approve & Activate order), so a strict `gate_clause` +mirror would blank a PI's conversations feed the moment their bot goes live, +before anyone had a chance to misconfigure anything. + +`own_or_gated` widens `gate_clause` with an explicit own-post carve-out: the +viewing agent's own rows always render, regardless of gate. This is safe +because the OR's second arm is keyed on `agent_id == agent_id` — the *viewing* +agent's own id, fixed by the route's own authorization +(`get_agent_with_access`), not attacker input — so it can only ever admit rows +this exact agent authored, never another agent's. It cannot be used to read +anyone else's traffic. + +Three call sites share one `own_or_gated(gate, aid)` expression rather than +three independently-written clauses, specifically so they cannot drift apart: +the conversations feed's roots query, its reply-count query, and the +thread-expand endpoint's root re-resolution and reply fetch (§5.1, §5.2). + +Covered by `test_an_uncohorted_agent_still_sees_its_own_posts` and +`test_expanding_an_uncohorted_own_thread_is_200_not_404` +(`tests/integration/test_conversation_feed.py`). + +## 5. Data flow + +### 5.1 Feed — `GET /agent/{agent_id}/conversations` + +The channel set is **unchanged and needs no gate**: it derives from the agent's own +authored messages (`agent_id == aid`). The unconditional `#general` union stays — +`#general` is the lobby, and it is now cohort-filtered like every other channel, +which is precisely what made it a leak before. + +The single flat query is replaced by two: + +1. **Roots** — `thread_ts IS NULL AND phase == "new_post" AND gate_clause(gate)`, + `ORDER BY posted_at DESC, created_at DESC, id DESC LIMIT 50`, then reversed for + oldest-first render. + + The three-column ordering must survive **verbatim**, along with its comment at + `agent_page.py:761-769`: migration 0019 added `posted_at` with `server_default + '0'`, so every pre-migration row shares one value, and `ORDER BY posted_at DESC + LIMIT n` over a tie group larger than `n` lets Postgres return any `n` — measured + on a 200-row tie group, the index-scan and seq-scan plans returned two **disjoint** + pages. + + `phase == "new_post"` is belt-and-braces alongside `thread_ts IS NULL`; the two + agree on every current row (snapshot 2026-08-05: 304 messages, 273 roots, 31 + replies; `phase` partitions them identically). + + Note this changes the window's **unit**: today it is the newest 100 *messages*, + and it becomes the newest 50 *threads*. Threads are the thing a PI reads, and + replies no longer consume window slots, so 50 roots surfaces strictly more + distinct conversations than 100 mixed rows did. + +2. **Reply counts** — `thread_ts IN (root message_ts) AND gate_clause(gate)`, + `GROUP BY thread_ts`. Gated, so the badge equals what expansion renders. + Precedent: `src/routers/admin.py:537-545`. + +**The gate goes into SQL before `LIMIT`, never in Python after it.** `#general` +carries traffic from 56 out-of-cohort labs; post-`LIMIT` filtering would let it +consume the window and leave a spoke PI with a near-empty page. + +The row dict gains `message_ts` and `reply_count`. + +### 5.2 Expand — `GET /agent/{agent_id}/thread/{message_ts}` (new) + +Returns a **server-rendered HTML fragment**, not JSON: no `fetch()` exists anywhere +in this codebase yet, and a Jinja partial matches house style and keeps rendering +logic server-side. + +Authorization is four server-side checks: + +1. `get_agent_with_access(agent_id, db, current_user)` — owner or delegate, else + 403 (`src/dependencies.py:94-127`). +2. Root exists in the **current run** and has `thread_ts IS NULL`. +3. Root's `channel_name` is in **this agent's** channel set. +4. Root passes `gate_clause(gate)`. + +Any failure returns **404**. Checks 2–4 are the IDOR defense: `message_ts` is +otherwise a guessable identifier that would read out any thread in the run. + +Replies are then fetched with the same gate, `ORDER BY posted_at ASC` (with +`created_at`, `id` tiebreakers for the same reason as §5.1). + +### 5.3 Client + +Vanilla JS, matching `templates/agent/public_profile.html:245-307`: a click handler +per root that fetches the fragment, injects it once, caches it, and toggles +thereafter. No framework, no client-side templating. + +### 5.4 Known limitation (accepted) + +Roots order by their **own** `posted_at`, so a new reply does not bump an old thread +back into the newest 50. At 273 roots / 31 replies this is not observable. Fixing it +requires a correlated `max(reply.posted_at)` in the `ORDER BY`; deferred as not worth +the cost now. Reviewed and accepted during brainstorming. + +## 6. Component: topology payload + +### 6.1 Template + +Replace the per-cell hidden input (`cohort_topology.html:82`) with per-row and +per-column markers emitted once each: + +```html +{% for a in agents %}{% endfor %} +{% for c in cohorts %}{% endfor %} +``` + +The checkbox at `cohort_topology.html:83` is unchanged. + +### 6.2 Handler + +`admin.py:1559-1569` rebuilds `rendered` as the cross product: + +```python +form = await request.form(max_fields=_TOPOLOGY_MAX_FIELDS) +present_agents = {v for v in form.getlist("present_agent") if isinstance(v, str)} +present_cohorts = {v for v in form.getlist("present_cohort") if isinstance(v, str)} +rendered = {f"{cid}:{aid}" for cid in present_cohorts for aid in present_agents} +``` + +Everything downstream (`admin.py:1571-1626`) is untouched: the `ticked - rendered` +malformed-submission check, the unknown-id skip, the per-cell add/remove diff, and +the per-change audit events. + +**60 + 56 = 116 markers** (+168 ticked = 284 fields, down from 3,528). + +### 6.3 Why the safety property is preserved + +The existing guarantee is that a stale or partial form cannot delete memberships for +a cohort or agent it did not display. That holds because `rendered` is exactly the +set of displayed cells — and the displayed cells **are** the cross product of +displayed rows and displayed columns, since the template renders a full matrix +(`cohort_topology.html:61-88`, an unconditional nested loop). Reconstructing the +product server-side yields an identical set, so the property is unchanged rather +than merely approximated. + +`max_fields` is raised alongside as a guard, because ticked cells alone will +eventually pass 1,000 even with the restructure. + +## 7. Testing + +Gate is `./scripts/ci.sh`: alembic sanity → `ruff check` on the suite → full pytest +with a branch-coverage floor. + +### 7.1 Parity test (the important one) + +Table-driven unit test over representative rows — human; bot in-cohort; bot +out-of-cohort; bot with NULL `agent_id`; `collab_private`; gate `None`; gate `set()` +— asserting that `_entry_allowed` and `gate_clause` return **the same verdict on +every row**. This is what stops the two implementations drifting, which is the exact +failure mode `src/services/cohorts.py:1-8` was written to prevent. + +### 7.2 Integration — feed + +Against the star topology: a spoke PI cannot see another spoke's bot in `#general`; +hub bots remain visible to every spoke; the PI's own messages still render +(preserving `tests/integration/test_agent_page.py:906-911`); an uncohorted agent +under `policy="isolated"` sees humans only; a delegate sees exactly what the owner +sees. + +This closes a real coverage gap: `test_agent_page.py` currently has **no** test +asserting which *other* agents' messages appear, unlike the dashboard test at +`test_agent_page.py:876-879`, which explicitly controls for its `agent_id` filter. + +### 7.3 Integration — threads + +Reply badge count equals the number of rows the expansion renders; expanding a +`message_ts` from a channel the agent does not participate in returns 404; +expanding an out-of-cohort root returns 404; expanding a reply's `message_ts` +(not a root) returns 404; a non-owner non-delegate gets 403. + +### 7.4 Integration — topology + +Round-trip save with the new payload adds and removes the expected memberships; a +form that omits a column cannot delete that column's memberships; a form that omits +a row cannot delete that row's memberships; an explicit assertion that a full-matrix +POST stays under `max_fields`. + +## 8. Migration / operational notes + +No schema change; no alembic revision. `message_ts`, `thread_ts`, `phase`, and +`visibility` all already exist and are populated. + +No agent-run restart is required — this is web-tier only, and touches no module the +`agent-run` process loads. Per `CLAUDE.md`, `docker compose up -d --build app` is +sufficient to pick the change up. + +Grouping must key on `thread_ts` / `message_ts` and **never** on `slack_thread_ts`: +the two diverge whenever a thread started with Slack off, because the canonical root +id is then locally minted and is not a valid Slack ts +(`src/agent/message_log.py:35-40`, `specs/local-db-conversations.md:37-38`). diff --git a/docs/specs/2026-08-06-hub-budget-scheduler-design.md b/docs/specs/2026-08-06-hub-budget-scheduler-design.md new file mode 100644 index 0000000..292117e --- /dev/null +++ b/docs/specs/2026-08-06-hub-budget-scheduler-design.md @@ -0,0 +1,333 @@ +# Design — load-proportional budget and scheduling for star topologies + +**Status:** DESIGN, not implemented. +**Date:** 2026-08-06 +**Target branch:** `blackbird` +**Closes:** A7 (per-role caps/budgets), explicitly deferred by +`docs/specs/2026-08-05-hub-bot-customization-design.md` §2 and left open in +`docs/blackbird-star-topology-runbook.md`. +**Companion:** `docs/blackbird-star-topology-runbook.md` (the star topology). + +--- + +## 1. Problem + +The blackbird deployment is a star: one `blackbird` hub bot and 56 PI spokes, wired as +56 pairwise cohorts. The hub is one endpoint of *every* conversation; each spoke is an +endpoint of one. + +Both the LLM budget and the turn scheduler treat all agents as interchangeable. For a +star, that is not neutral — it is actively wrong, and it took the hub off the air. + +### 1.1 What actually happened (measured, run `4f1e8395`, 2026-08-05) + +The simulation was started with `--budget 40`. `_agent_within_budget` +(`src/agent/simulation.py:375`) is `api_call_count < budget_cap`, and `_turn_eligible` +drops any agent failing it from `_select_agent`'s candidate pool entirely. + +| Agent | LLM calls | +|---|---| +| `blackbird` | **42** | +| `scott`, `bailey`, `tsapatsis` | 9 | +| next 8 agents | 7–8 | + +The hub was the only agent at or over the cap. It went silent at 21:09 and took +**0 of the 161 turns** in the following 2.5 hours while every other agent took 3–5. + +Two properties made this worse than a normal cap: + +- **It is silent.** Nothing logs when an agent is benched. The failure presents as "the + bot stopped talking", with no error anywhere. +- **It survives restarts.** `_rebuild_state` step 4 (`simulation.py:~3822`) recomputes + `api_call_count` with a `COUNT(*)` over `llm_call_logs` for the *same* + `simulation_run_id`. A container restart therefore restores the agent to its + over-budget state. Only `--budget 0` or `--fresh` clears it. The 22:43 restart could + never have helped, and did not. + +### 1.2 Why the hub burns calls faster (structural, not a bug) + +Servicing one spoke costs the hub roughly four LLM calls: Phase 2 `scan`, Phase 4 +`thread_reply`, Phase 5 `new_post`, and the thread-closure working-memory update +(`simulation.py:~4670`). Its 42 calls decomposed as new_post 14, scan 13, thread_reply +8, memory 7 — only 19% were replies. At ~4 calls per spoke, `--budget 40` buys ~10 +spokes out of 56. It reached 8. + +This is not fixable by raising the number. Any uniform cumulative cap is a countdown to +the same bench; raising it only moves the cliff. + +### 1.3 The scheduler shortfall is separate, and real + +Before the bench, the reactive tier was already working: the hub took **12.2% of all +LLM calls against a 1.75% equal share** — a 7× boost. But in a star the hub is one +endpoint of every conversation, so a healthy hub should approach ~50% of traffic. It +got 12.2% of calls and 5% of messages (16 of 321). + +There is also a specific defect. The reactive tier breaks ties with +`min(owed, key=lambda a: a.state.last_selected)` (`simulation.py:~705`). The hub is +selected often, so its `last_selected` is always recent, so it **loses every tiebreak to +a long-idle spoke**. The scheduler penalizes the hub precisely for being busy. + +### 1.4 Root cause + +The limiter and the scheduler hold **contradictory models of what the hub deserves**. +The reactive tier said "act, you owe 8 replies"; the cumulative cap said "you are +done" — and the cap won, silently. Any fix that patches one side leaves the +disagreement in place. + +## 2. Requirements (settled during brainstorming) + +- **`--budget` is a circuit breaker and a pacing lever, NOT a cost cap.** Confirmed with + the operator. Total spend is not what it protects; runaway behaviour and + conversational pacing are. This is why "give the hub a bigger number" is the wrong + shape of answer. +- **The limit becomes a rate (sliding window), not a cumulative total.** A rate serves + both goals directly, and self-heals: a throttled agent becomes eligible again as the + window slides. No permanent bench, no sticky-across-restart failure. +- **Allowance and scheduling weight derive from one shared load signal**, so the two + can never disagree again (§1.4). +- **Scope is budget + scheduler.** Full observability tooling (dashboards, per-agent + turn-share reporting) is out of scope; one throttle-transition warning is in (§6). + +### 2.1 Non-goals + +- No cost/token budgeting. Explicitly rejected by the operator. +- No change to `active_thread_threshold`, thread turnover, or hub coverage policy. See + §7 — after this fix, coverage is bounded by thread capacity, and that is intended to + remain a separate, visible knob. +- No schema change, no migration. +- No fix for the hybrid-Slack problem (hub has no bot token, so its posts are + `MOCK post` and never reach Slack). Separate issue, tracked in the runbook. + +## 3. Was edge-based budgeting the answer? + +Directionally yes, and the arithmetic checks out: if each spoke runs at rate `R`, total +spoke traffic is `56R`, and the hub — the other endpoint of all of it — needs ≈`56R`. +An allowance proportional to edge count is dimensionally correct. + +It was rejected as the *implementation* for two reasons: + +1. **Edges ≠ live load.** The hub has 56 cohort edges but is bounded to + `active_thread_threshold` (12) concurrent threads. Budgeting on 56 over-allocates by + ~5×. +2. **It weakens the breaker where it matters most.** A 56× allowance hands the largest + blast radius to the agent with the biggest prompt, the most tools, and the most + complex role — the one most likely to run away. + +Load-proportional budgeting keeps the correct dimension while tracking reality and +keeping the breaker tight. It is edge-based budgeting with the right denominator. + +## 4. Design + +### 4.1 The load signal + +One method on `SimulationEngine`: + +```python +def _agent_load(self, agent: Agent) -> int: + """Concurrent conversational obligations. 1 for an idle agent.""" + live = sum(1 for t in agent.state.active_threads.values() if t.status == "active") + return max(1, min(live, get_settings().active_thread_threshold)) +``` + +The clamp is load-bearing at both ends. The floor of 1 keeps an idle agent eligible. +The ceiling means nothing can inflate its own allowance past the thread cap it is +already bound by — which is what stops a thread-opening runaway from financing itself. + +### 4.2 Consumer 1 — the rate limiter + +`AgentState` gains `call_times: deque[float]`. Every existing `agent.api_call_count += 1` +site (`simulation.py:857, 903, 1172, 1947, 4670`) is replaced by a single +`agent.record_api_call()` helper maintaining both counters. `api_call_count` is retained +— it is reported in the run summary and in `SimulationRun.total_api_calls`. + +A new `_within_rate_limit` performs the window check: + +```python +def _within_rate_limit(self, agent: Agent, now: float) -> bool: + settings = get_settings() + allowance = self._calls_per_load(agent) * self._agent_load(agent) + window_start = now - settings.llm_rate_window_seconds + while agent.state.call_times and agent.state.call_times[0] < window_start: + agent.state.call_times.popleft() + return len(agent.state.call_times) < allowance +``` + +where `_calls_per_load` returns the role override (§4.4) if set, else +`settings.llm_calls_per_load_per_window`. + +`_agent_within_budget` is **retained, not replaced** — it is the legacy cumulative cap +(§6) and is now inert by default, since `budget_cap` defaults to 0 and it already +short-circuits to `True` at `<= 0`. `_turn_eligible` requires **both** checks to pass: + +```python +return ( + self._agent_within_budget(agent) # legacy, inert unless --budget is passed + and self._within_rate_limit(agent, now) + and cooldown_ok +) +``` + +The cooldown branch is unchanged. The main loop's redundant second +`_agent_within_budget` call at `simulation.py:502` is left alone; it is unreachable-false +given `_select_agent` only returns eligible agents, and removing it is out of scope. + +**Restart behaviour.** `_rebuild_state` step 4's all-time `COUNT(*)` into +`api_call_count` is **left exactly as is** — that counter still feeds the run summary +and `SimulationRun.total_api_calls`, and an existing integration test +(`test_full_run_live.py:1111`) correctly pins its survival across restart. A new +**step 4b** additionally selects `created_at` for rows **inside the window** and loads +them into `call_times`. + +The two counters therefore mean different things on purpose: `api_call_count` is +lifetime accounting, `call_times` is the live throttle. Because only the latter gates +eligibility and its entries age out, §1.1's sticky bench becomes impossible by +construction rather than by correct operator behaviour. + +### 4.3 Consumer 2 — the scheduler + +Two changes in `_select_agent`, both using the same `_agent_load`: + +- **Proactive tier:** `w = max(now - a.state.last_selected, 1.0) * self._agent_load(a)`. + The existing Phase-5 skip penalty is unchanged. +- **Reactive tiebreak:** replace `min(owed, key=lambda a: a.state.last_selected)` with + `max(owed, key=lambda a: (now - a.state.last_selected) * self._agent_load(a))`. + Still "longest wait wins", now scaled by obligation count, which fixes §1.3. + +`max_consecutive_reactive_turns` is unchanged; the fairness valve still applies. + +### 4.4 Optional per-role override + +`RoleSpec` gains `calls_per_load_per_window: int | None = None`, read from an optional +`role.toml` key of the same name. When set it overrides the global setting for that +role. Malformed or non-positive values are logged and ignored, matching `load_role`'s +existing never-raises contract. + +This exists to pin a specific agent when needed. It is **not** the mechanism — the load +signal is. No role sets it initially, including `scout_hub`. + +## 5. Configuration + +Two new `Settings` fields in `src/config.py`: + +```python +llm_rate_window_seconds: int = 600 # sliding window +llm_calls_per_load_per_window: int = 8 # allowance per unit of load +``` + +Both are `@lru_cache`d via `get_settings()`, so like the cohort flags they require a +container **recreate**, not a restart. + +Calibration against measured rates from run `4f1e8395`: + +| | observed | allowance | headroom | runaway trip time | +|---|---|---|---|---| +| Spoke (load 1) | ~0.27 calls/10min | 8/window | ~30× | ~25s | +| Hub (load 12) | ~2.6 calls/10min | 96/window | ~37× | ~5 min | + +**Known trade-off:** a runaway *hub* takes ~5 minutes to trip, versus ~25 seconds for a +spoke. That is the direct price of the 12× allowance. Lower +`llm_calls_per_load_per_window` to tighten it; the spoke headroom is large enough to +absorb a reduction to 4 without risk. + +## 6. Back-compat and failure modes + +**`--budget` is deprecated, not removed.** Its default changes `50 → 0` (off). A nonzero +value is still honored as a hard cumulative cap, but logs a prominent warning naming it +as the legacy mechanism that benches hubs. Deleting the flag would break operator muscle +memory and the CLAUDE.md runbook; leaving it silently armed would let §1.1 recur the +next time someone types `--budget 40`. CLAUDE.md's "Running the Agent Simulation" +section is updated in the same change. + +**Throttle visibility.** A `WARNING` is logged when an agent *transitions* into +throttled state — once per transition, not per turn. This is a deliberate, small +incursion into the observability scope that was otherwise cut: a silent throttle is +precisely what turned this into a 2.5-hour undetected outage. + +**Failure modes considered:** + +- *All agents throttled simultaneously.* `_select_agent` returns `None`. The loop used + to **break** here, which under a sliding-window limiter is wrong: throttling and the + per-agent `turn_delay_seconds` cooldown both lapse with time, so breaking converts a + temporary bench into a permanent whole-run stop — strictly worse than the failure this + design replaces, and reachable on any roster small enough for aggregate demand to meet + aggregate allowance. The loop now consults `_terminal_stall_reason()`: it logs, applies + the shared idle backoff and **continues**, and breaks only for the two conditions that + cannot recover — an empty roster, or the legacy `--budget` cap armed (`> 0`) and blown + by *every* agent. `max_runtime` and SIGTERM still end the run via the loop condition, + and the backoff sleep (which returns early on stop) is what keeps this from spinning. +- *Clock skew / non-monotonic time.* `call_times` uses `time.time()`, consistent with + `last_selected` and `last_phase5_action_time`. A backwards jump can only delay + pruning, never bench an agent permanently. +- *Empty `active_threads` at startup.* Load floors at 1, so a cold agent is eligible. + +## 7. Consequence to accept deliberately + +With `active_thread_threshold=12`, the hub's allowance and scheduling weight cap at +**12×, not 56×**. That is intended: 12 is the number of conversations it can actually +hold. + +After this change, spoke coverage is bounded by **thread capacity and turnover**, not by +budget. If 8-of-56 coverage remains too thin, the next lever is +`ACTIVE_THREAD_THRESHOLD` — a separate, visible knob — not the budget. Hiding hub +capacity inside a limiter is how the original problem became invisible. + +## 8. Testing + +New unit tests in `tests/unit/`, using the existing `_engine` helper +(`tests/unit/test_cohort_isolation.py:109`): + +- `_agent_load`: idle → 1; N active threads → N; clamped at `active_thread_threshold`; + non-active threads excluded. +- Rate limiter: under allowance → eligible; over → ineligible; **eligible again once the + window slides**. This is the regression test for the permanent bench. +- Restart rebuild: `call_times` repopulated only from `llm_call_logs` rows inside the + window; an agent whose calls all predate the window starts unthrottled. +- Scheduler, proactive: seeded statistical test that a load-12 hub receives ≈12× a + spoke's selection share. +- Scheduler, reactive: a busy hub beats a long-idle spoke when load justifies it + (direct regression for §1.3). +- Role override: `calls_per_load_per_window` honored when set; ignored and logged when + malformed or non-positive. +- **Production regression**, promoted from the reproduction script written during + diagnosis, reconstructing the exact §1.1 state (hub at 42 calls, 56 spokes at 8). + Three assertions, because the legacy cap is retained (§6) and the three cases differ: + 1. Under the new default (`budget_cap=0`) with those 42 calls **outside** the window: + the hub is selectable. This is the fix. + 2. Under the new default with 42 calls **inside** the window: the hub is throttled, + then becomes selectable once the window slides. Throttling is still real — it just + is not permanent. + 3. With `--budget 40` explicitly passed: the hub is still benched, and the + deprecation warning is emitted. This pins the compat path honestly rather than + pretending the legacy flag was made safe. +- Composition: `_turn_eligible` fails if *either* the legacy cap or the rate limit + fails, and passes only when both do. + +**Existing tests — one changes, one must NOT:** + +- `tests/unit/test_cohort_isolation.py:1175 test_budget_still_filters` sets + `budget_cap=1` explicitly, so it still exercises the retained legacy cap and + **passes unchanged**. It is extended, not rewritten, with a sibling asserting the + rate limiter filters independently of `budget_cap`. +- `tests/integration/test_full_run_live.py:1111` asserts `api_call_count` survives + restart. Per §4.2 that counter is deliberately untouched, so this test **must keep + passing with no edit**. If it fails, step 4b has been implemented by modifying step 4 + rather than adding to it — treat a failure here as the intended tripwire, not as a + test to update. + +Called out explicitly because "the tests changed" is where a fix of this shape can hide +a regression. + +**Gate:** `./scripts/ci.sh` must stay green — single alembic head, `ruff` clean on +tests, `src/` findings at or under `SRC_LINT_MAX=260`, branch coverage at or above +`COV_MIN=60`. + +## 9. Out of scope / follow-ups + +- Hub has no `slack_bot_token`; with `SLACK_ENABLED=true` its posts take the `MOCK post` + branch and never reach Slack (0 of 16 messages carried a `slack_ts`). Independent of + this design. +- `_build_lab_directories` is not cohort-aware, inflating every system prompt with all + other labs' publications (hub `new_post` averaged 23,873 input tokens). A cost issue, + not a correctness one. +- Hub coverage of all 56 spokes (§7) — needs a thread-capacity decision, not a budget + one. diff --git a/docs/specs/2026-08-06-role-topology-post-type-gating-design.md b/docs/specs/2026-08-06-role-topology-post-type-gating-design.md new file mode 100644 index 0000000..b38c684 --- /dev/null +++ b/docs/specs/2026-08-06-role-topology-post-type-gating-design.md @@ -0,0 +1,552 @@ +# Role- and topology-aware post-type gating + +**Date:** 2026-08-06 +**Status:** Design, approved. Not implemented. +**Branch:** `blackbird` + +> **org1 note (2026-08-10).** This branch ports the *machinery* described below — +> `src/agent/post_types.py`, the `role.toml` `post_types` key, and the +> `{post_type_menu}` substitution — but **not** the enforcement in §"Layer 3" or the +> phase-5 rejection call. `66948dc` is deliberately excluded: org1 runs a mesh with +> `cohort_isolation_enabled=False`, where layers 2 and 3 are inert and layer 1 buys +> nothing, and its `prompts/phase5-new-post.md` carries no `{post_type_menu}` token, so +> no menu renders. Enable enforcement when cohorts are turned on, together with a +> purpose-built org1 prompt variant, as its own change with its own measurement. See +> `docs/specs/2026-08-10-org1-parity-design.md` §7.1. + +## 1. The problem, measured + +In the live star topology, 56 cohorts each hold `{, blackbird, grantbot}`, so no `pi_lab` +agent may interact with any other `pi_lab` agent. Yet in the latest simulation run: + +| Metric | Value | +|---|---| +| `:bulb:` Idea top-level posts | 259 | +| …carrying the tag-strip artifact (`Idea —,`) | 200 (77%) | +| …leaking an `@agent_id`-style cross-cohort tag to Slack | 12 | +| …naming the lab in prose only, no tag | 42 | +| …mentioning `blackbird`, the one reachable partner | **0** | +| …that received any reply | **2 (0.8%)** | +| `:newspaper:` Paper posts (control) | 234, **0** artifacts, 21 replies (9.0%) | +| Phase-5 posts declaring `tagged_agent` (13h container run) | 146 | +| …targeting `blackbird` or `grantbot` | **0** | + +The hub's interview pipeline is seeded 20 threads from `:newspaper:` Paper versus 2 from +`:bulb:` Idea. Ideas are 53% of top-level posts and 9% of the pipeline. + +### Root cause chain + +1. **`_build_lab_directories` runs before the cohort gate exists.** The filter at + `src/agent/simulation.py:3663` is correct, but `start()` calls the builder at `:508` and + `_recompute_allowed_sender_ids()` only at `:533`. Every gate is still `None` + (`src/agent/agent.py:85`), so the filter no-ops. On a no-change roster tick the gate is + recomputed (`:4551`) but the directory rebuilds only `if role_changed` (`:4549`) — so it is + never rebuilt against a live gate. Verified: zero roster add/remove and zero role changes in + the 13-hour run. + + Production confirmation from a stored `llm_call_logs.system_prompt`: `gill`'s phase-5 prompt + names **51 labs**, all unreachable, and the string "Blackbird" appears **nowhere**. The + directory is **69%** of the 67 KB system prompt. + + This is runbook gap **A3**, which + `docs/specs/2026-08-05-hub-bot-customization-design.md:261` records as closed. The guard was + written; the call ordering makes it dead code. + +2. **The prompt then demands a tag.** `prompts/phase5-new-post.md:129-131` — "TAG the other + lab's agent (e.g., @WisemanBot)". + +3. **`tagged_agent` is never validated** — only logged (`simulation.py:2424`). + +4. **The mention is stripped and the post ships anyway.** `_strip_disallowed_tags` + (`:2539`, applied from `_post_message` at `:3359`) removes it, `_post_message` (`:3299`) posts the remainder. End-to-end from + production: `{"post_type":"idea_crosslab","tagged_agent":"pearce"}` with body + `:bulb: Idea — @PearceBot, your recent finding…` became `:bulb: Idea —, your recent finding…`. + +5. **The strip regex requires the `Bot` suffix** (`:2596`), so `@pearce`-style tags bypass it + entirely — the 12 leaked tags. + +6. **Strips are logged at DEBUG** (`:2585`) under `level=INFO` (`src/agent/main.py:23`), so 200 + of them produced no operator-visible signal. + +### Why prompt text cannot fix this + +`post_type` is read once (`:2203`) and compared twice — `== "funding_collab"` (`:2230`), +`== "opportunity_assessment"` (`:2383`). No enum, no allow-list, no rejection, never persisted. +Two consequences today: a blocked agent can self-declare `funding_collab` to bypass the +proposal block, and any `pi_lab` agent declaring `opportunity_assessment` writes an +`OpportunityAssessment` row with no role check. + +More fundamentally, the constraint is **topological, not role-intrinsic**: `pi_lab` in org1's +mesh *should* make cross-lab idea posts — that is the product. The same role in this star must +not. A prompt cannot know which deployment it is in. + +## 2. Canonical vocabulary + +Today's vocabulary cannot support an allow-list. `idea` and `idea_crosslab` are both in the +enum (`prompts/phase5-new-post.md:169`) with no documented difference and no code +distinguishing them; `:question:`, `:test_tube:` and `:package:` are offered as labels +(`prompts/agent-system.md:219-221`) with no `post_type` at all; `:memo:` and `:mag:` are the +most consequential markers and are absent from the enum. + +| `post_type` | Emoji | `targets` | pi_lab | scout_hub | +|---|---|---|---|---| +| `paper` | `:newspaper:` | — | ✓ | | +| `help_wanted` | `:sos:` | — | ✓ | | +| `introduction` | `:wave:` | — | ✓ | | +| `idea_crosslab` | `:bulb:` | `["pi_lab"]` | ✓ | | +| `pitch` | `:bulb:` | `["scout_hub"]` | ✓ | | +| `funding_collab` | `:moneybag:` | `["pi_lab"]` | ✓ | ✓ | +| `opportunity_assessment` | `:mag:` | — | | ✓ | + +Resolutions: collapse `idea` into `idea_crosslab`; add `pitch` (a spoke pitching its own +commercializable idea to the hub); drop `:test_tube:` and `:package:` from the `pi_lab` label +table (unused, undefined); mark `:question:` reply-only, which both roles' prompts already +imply. `:memo:` stays outside this vocabulary — it is a thread-reply marker, not a top-level +type. + +**Scope: the allow-list governs `action: "new_post"` only.** `action: "reply"` is untouched. + +## 3. Architecture — three layers + +New module `src/agent/post_types.py`, dependency-free like `roles.py` and `thread_guidance.py`, +so the filter is unit-testable without a DB, an engine, or a running loop. + +### Layer 1 — declarative allow-list in `role.toml` + +Mirrors the existing `tools` key and is enforced the two ways `tools_for_role` is +(`src/agent/tools.py:128` filters what the model sees; `:148` refuses at dispatch). + +Shape of the key (illustrative — `targets` names the `AgentRegistry` roles a type may +address; an EXPLICIT empty list means the type addresses no one, while an ABSENT key +instead inherits that type's CANONICAL default, which may not be empty — `paper` here +has no `targets` line only because its canonical default already is empty): + +```toml +[[post_types]] +name = "paper" + +[[post_types]] +name = "idea_crosslab" +targets = ["pi_lab"] + +[[post_types]] +name = "pitch" +targets = ["scout_hub"] +``` + +An absent `post_types` key yields `DEFAULT_POST_TYPES`, explicit for the same reason +`DEFAULT_TOOLS` is (`roles.py:22-27`): a newly added type must stay opt-in rather than being +handed to every role silently. + +`pi_lab` has no `role.toml` today — "pi_lab is the absence of overrides" (`roles.py:58-60`). +`DEFAULT_POST_TYPES` therefore *is* pi_lab's list, and `prompts/roles/pi_lab/` still need not +exist. + +### Layer 2 — topology filter + +Keep a post type iff its `targets` is empty, **or** there exists an agent in the acting +agent's `allowed_sender_ids`, excluding itself, whose `AgentRegistry.role` is in `targets`. + +| Gate state | Behaviour | +|---|---| +| `None` (mesh, or isolation off) | **No filtering by reachability.** A `targets` type is still dropped when no agent of a matching role exists on the roster — that is what removes `pitch` in a hubless mesh. Layer 3 is skipped outright. | +| Set | Filter as above | + +The `None` case is what preserves org1's mesh byte-for-byte, and it is the same way every +other cohort feature degrades. + +Worked results: + +- **Star.** `gill`'s gate is `{gill, blackbird, grantbot}`. Excluding self, roles are + `blackbird=scout_hub` and `grantbot=`(no `AgentRegistry` row → unknown). No `pi_lab` peer, so + `idea_crosslab` and `funding_collab` drop; `blackbird` is `scout_hub`, so `pitch` appears. +- **Mesh.** Lab peers exist, so `idea_crosslab` stays; no `scout_hub` exists, so `pitch` drops + automatically with no per-deployment configuration. + +An unknown counterparty role matches no `targets`. That is correct for `grantbot`, which is a +funding announcer, not a pitch recipient. + +### Layer 3 — `tagged_agent` validation + +Reject the post when `tagged_agent` is set and either is not in `allowed_sender_ids`, or its +role is not in the chosen type's `targets`. This is the layer that would have stopped all 146 +posts, and it closes the `@pearce` bypass because it validates the JSON field rather than +scanning prose. + +Three sub-rules, each chosen so the failure mode is proportionate: + +- **A tag on a broadcast type is not an error if the tag is reachable.** `opportunity_assessment` + addresses no one, but it is posted into the PI's own channel and the natural thing for the model + to do is name that PI. Rejecting it would destroy the most valuable artifact in the system — + and the whole interview behind it — over a field that is merely redundant. The tag is simply + ignored for routing; the text mention survives because the PI is in the hub's gate. An + *unreachable* tag on a broadcast type is still a rejection. +- **A `targets` type with `tagged_agent: null` is a rejection.** An addressed post that addresses + no one is exactly the dangling ask this work exists to stop. +- **Layer 3 is fully inert when the gate is `None`.** Not "inert for reachable agents" — skipped + entirely, including the null check. A mesh deployment's phase-5 behaviour must be + byte-identical after this change, and today a hallucinated `tagged_agent` there is logged and + the post ships. Tightening that is a separate decision, not a side effect of this one. + +### Enabling fix — directory ordering + +Make the directory a *derived product of the gate*: `_recompute_allowed_sender_ids` refreshes it +on every path it takes, including the two that disable gating. The three existing +`_build_lab_directories()` call sites (`:508`, `:4549`, `:4594`) then reduce to one — the +role-change branch, where the directory's contents move without the gate moving. + +An earlier draft proposed keying the rebuild on the gate signature `_recompute_allowed_sender_ids` +computes at `:4690`. Rejected: that signature is `(cohort_count, len(rows), gated_count, +isolated)`, which is a *logging* fingerprint, not a per-agent one — two topologies with the same +counts but different memberships share it. Refreshing unconditionally is O(agents) over in-memory +profiles with no I/O, on a 30-second cadence. + +Without this, layer 2 filters the menu correctly while the prompt still advertises 51 +unreachable labs, and the model keeps working that roster as a backlog. Its own reasoning shows +this: *"I've already covered … Srinivasan/malaria, Weeraratna/melanoma … Let me look at labs I +haven't engaged with yet."* + +### Enabling fix — the `funding_collab` bypass must not survive on the reply path + +§1 records that a blocked agent can self-declare `funding_collab` to bypass the proposal block. +Layers 1–3 do not close it, because they govern `action: "new_post"` only and the bypass at +`simulation.py:2230` reads `post_type` regardless of action: + +```python +is_funding_post = post_type == "funding_collab" +``` + +So `{"action": "reply", "target_post_id": , "post_type": +"funding_collab"}` still walks past the block. The fix is one clause — `action == "new_post" and +post_type == "funding_collab"` — and it belongs in this change rather than a later one, because +this is the change that makes `post_type` a load-bearing, enforced field. A funding *reply* is +already covered by `is_funding_reply` on the line above, which checks the thread rather than the +model's self-declaration. + +### The menu must never enumerate an empty list + +`render_menu` describes each available type and, for a type with `targets`, names the agents it +may address. When the gate is `None` there is no enumeration to make — the mesh has 50 reachable +labs and listing them in every phase-5 prompt would recreate the 46 KB lab directory this design +is shrinking. So the addressed-type line has two forms: + +| Gate | Rendering for a type with `targets` | +|---|---| +| Set | "Set `tagged_agent` to exactly one of: …" — the enumerated reachable agents | +| `None` | Guidance only: address one agent of the matching role, by its `agent_id` | + +This is not only a mesh nicety. Without it, the "no topology supplied" default path renders +`Set tagged_agent to exactly one of: .` — an empty enumeration — and that string reaches a +committed characterization snapshot, because `test_phase5_prompt_gm` calls `build_phase5_prompt` +with no menu. + +### Prompt changes + +- A new `## Post types available to you this turn` section carries a `{post_type_menu}` token, + placed immediately before `## Instructions`. Option C's body defers to it instead of hardcoding + four types. + + **Single source of truth.** The menu is rendered from, and enforcement uses, one computed set: + the role's declared `post_types`, filtered by layer 2, then further restricted when the agent is + blocked for regular posts. Menu and enforcement cannot drift because they are one value — + `post_types.available_for(...)`. + + **The restriction is keyed on `blocked_for_regular`, not on `funding_only`.** Those differ: + `funding_only = blocked_for_regular and not has_available_non_funding` (`simulation.py:2107`), + so a blocked agent that *does* have a non-funding post available gets `funding_only=False`. + Keying the menu on `funding_only` would advertise `paper` and `pitch` to that agent and then + have the block at `:2230` reject the post anyway — the exact prompt-versus-enforcement + disagreement this design removes, reintroduced under a new name. `funding_only` continues to + drive the template surgery; only the menu/enforcement set uses `blocked_for_regular`. + + The `### Option C: Make a new top-level post` heading and the intro paragraph stay + byte-identical so `funding_only`'s existing regex surgery (`src/agent/agent.py:599-634`) still + matches. Verified against the drafts: all four surgeries and the intro replacement match, and + after the surgery the menu token survives while Option C is stripped and Option B kept. The + new token is added to the raw-template pins at `tests/unit/test_roles.py:160-227`. +- Introduce the hub in the `pi_lab` prompt. Required by the `pitch` type: "Blackbird" currently + appears nowhere in a spoke's prompt, profile, or directory. +- **Give `pitch` its own quality bar and its own worked example.** The existing bar asks for "a + specific dataset, technique, or reagent each lab would contribute" and "a concrete first + experiment" — correct for `idea_crosslab`, unfollowable for a pitch, whose counterparty has no + bench and contributes nothing. One bar for two types is how the old prompt ended up asking for + a tag the topology forbade. +- **Restore the `paper` preference explicitly.** The old Option C called `:newspaper:` "the + PREFERRED post type — always consider sharing a paper first". Deferring Option C to the menu + drops that clause, and `paper` is the one type with a measured 9.0% reply rate against 0.8%. +- **`pi_lab` phase-4 needs a hub-thread exception.** A `pitch` opens a thread between a PI bot + and the hub, and the PI bot runs the `pi_lab` EXPLORE/DECIDE/CONCLUDE strings from + `src/agent/thread_guidance.py`, which tell it to build toward a `:memo:` Summary naming "what + each lab brings". Against a hub with no lab that is unfollowable. Those strings are pinned + byte-for-byte by the snapshots and must not be reworded, so the exception goes in + `prompts/phase4-thread-reply.md`. +- **Both roles' phase-2 scan filters must say "tags a specific agent *other than you*".** The + current wording — "tags a specific other agent … that post is directed at them, not at you" — + is the only thing between a `pitch` and being filtered out by its own recipient. Phase 3's + tag auto-activation (`simulation.py:1114`) is the primary delivery path and does not depend on + the scan, so this is belt-and-braces; it is one word, and the failure it prevents is the whole + feature silently not working. +- **`scout_hub` has no `phase2-scan-filter.md` or `phase2-prune.md` override**, so it runs the + `pi_lab` versions: "relevant to *your lab's* core expertise", "Papers *your own lab* authored", + "labs whose capabilities complement yours". The hub has no lab. This predates the change and + does not block it, but phase 2 is what decides which PIs get interviewed at all, so the drafts + include overrides. Separable — recorded here so the reviewer can drop them without unpicking + anything else. +- **Institution:** `prompts/agent-system.md:3` and `prompts/identity.md:2` say "Scripps + Research". Measured: 57 of 60 public profiles say "Johns Hopkins University"; the only + "Scripps Research Institute" is `alanjary.md`, the test bot being disabled; + `mukherjeeclavin.md` and `pearce.md` name no institution. So the identity line becomes + **institution-neutral** and lets the public profile — injected directly below it — carry that + fact. Strictly more correct than either literal and needs no new per-agent field. + + **Coupled code change:** `_DEFAULT_IDENTITY` (`src/agent/agent.py:753`) is a fallback that must + match `prompts/identity.md` verbatim, *including the absence of a trailing newline* — see the + comment at `:750` and `_compose_system_prompt`, which depends on exactly one blank line between + blocks. Edit both or neither. The draft preserves the missing trailing newline (verified with + `xxd`). + +Draft prompt files for review: `docs/specs/2026-08-06-post-type-gating-prompts-draft/`. That +directory mirrors `prompts/` and holds the **complete** set each role resolves — 7 changed `.md` +files plus `role.toml`, the 2 new `scout_hub` overrides, and verbatim copies of the 3 that do not +change — so +a reviewer reads a bot's whole instruction set rather than reconstructing it from diffs. Its +`README.md` is the change-by-change rationale. + +## 4. Data flow + +``` +LLM returns {action, post_type, tagged_agent} + │ + ├─ action == "reply" ─────────────────────────► unchanged path + │ + └─ action == "new_post" + ├─ L1: post_type in role's declared set? ─ no ─► reject + ├─ L2: targets satisfiable from gate? ─ no ─► reject + ├─ L3: tagged_agent in gate & role in targets? ─ no ─► reject + └─ yes → _post_message (existing path, unchanged) + +reject = no Slack call, no message_count++, WARNING log, + consecutive_phase5_skips++ +``` + +Placement: inside the existing `else:` (new top-level post) branch, before `_post_message`. Note +`consecutive_phase5_skips` is zeroed earlier in the handler, before the branch, so a rejection +must re-increment it. + +### ⚠️ Do not trust a line number in this document + +`blackbird` takes concurrent commits from other work, and `simulation.py` has moved twice during +this document's life alone: the original references were written before `f7a9f68`/`a247ed8` (2-3 +lines low), and then `e116feb`, `44f09be`, `c6943d4`, `f32a83e` and `517a564` moved everything after `:1366` again — by up to 46 lines. Re-verified at `517a564`: `agent.py`, `roles.py` and `prompts/` were untouched by all five, so only `simulation.py` numbers moved. +Every code reference here is also quoted verbatim; **the quote is the anchor, the number is +decoration.** If they disagree, the quote wins. + +Values re-verified at `517a564` (HEAD), and the command that re-derives them: + +| Symbol | Line | +|---|---| +| `_phase5_new_post` | 1956 | +| `blocked_for_regular = …` | 1983 | +| `funding_only = …` | 2126 | +| `build_phase5_prompt` call | 2128 | +| `post_type = action_data.get(…)` | 2203 | +| `is_funding_post = …` (the bypass) | 2230 | +| `_strip_disallowed_tags` call | 2254 | +| new-post `else:` branch | 2360 | +| `_strip_disallowed_tags` def / DEBUG / regex | 2539 / 2585 / 2596 | +| `_post_message` def | 3299 | +| `_build_lab_directories` def | 3638 | +| `_build_lab_directories()` call sites | 508, 4549, 4594 | +| `_recompute_allowed_sender_ids()` call sites | 533, 4551, 4604 | +| `_disable_all_gates()` call sites | 4633, 4675 | +| `_apply_cohort_gate_to_state()` call sites | 4639, 4677, 4712 | + +```bash +grep -n "def _phase5_new_post\|blocked_for_regular = \|funding_only = \|\ +build_phase5_prompt(\|is_funding_post = \|def _build_lab_directories\|\ +_build_lab_directories()\|_recompute_allowed_sender_ids()\|_disable_all_gates()" \ + src/agent/simulation.py +``` + +## 5. Error handling + +Every failure mode fails toward "post nothing" rather than "post something wrong", except +where that would silence a deployment. + +| Condition | Behaviour | +|---|---| +| No `post_types` in `role.toml` | `DEFAULT_POST_TYPES`, WARNING once | +| Malformed entry (not a table, or no `name`) | Drop that entry, WARNING, keep the rest | +| Unknown `post_type` name | Drop it, WARNING — mirrors `roles.py:99-103` for tools | +| `targets` names a nonexistent role | Type never offered, WARNING at load (catches typos) | +| Gate is `None` | Layer 2 does not filter; layer 3 is skipped entirely, including its null check | +| Layer 2 filters the menu to empty | Render an explicit "none available" menu and reject any `new_post`; do **not** skip the turn | +| Menu has a `targets` type but no gate to enumerate from | Render guidance, never `one of: .` — see §3 | +| Counterparty role unknown | Matches no `targets` | +| Broadcast type carrying a reachable `tagged_agent` | Ignore the tag, publish — see §3 layer 3 | +| Broadcast type carrying an unreachable `tagged_agent` | Reject | +| Malformed `role.toml` overall | Existing behaviour: log ERROR, use defaults, never raise | + +The empty-menu row deserves care. An earlier draft of this design said "skip the turn", which is +wrong: the menu governs `action: "new_post"` only, so skipping would also suppress a legitimate +`action: "reply"` — exactly the funding replies that are the only thing a blocked spoke can still +do. Instead the menu renders an explicit "no new top-level post type is available to you this +turn — reply or skip", and enforcement rejects only an actual `new_post`. + +In normal mode the menu cannot empty out anyway, because the broadcast types carry no `targets`. +It can and does empty for a blocked spoke in the star, where `funding_collab` is the only +new-post candidate and has no reachable `pi_lab` — which is precisely the case that must still +leave Option A open. + +## 6. Testing + +The bug survived because `tests/unit/test_simulation_logic.py:1104` pre-seeds +`allowed_sender_ids` by hand at `:1110-1115` before calling the builder, and +`tests/unit/test_roster_sync.py:108` stubs the builder out entirely. Priority is therefore +tests that exercise **production order**, not just the predicate. + +1. **Ordering regression** — the directory is gate-scoped after `start()`'s real sequence. + `start()` does too much I/O to drive in a unit test, so this is two tests, not one: a source + assertion that `_recompute_allowed_sender_ids` precedes the directory rebuild in `start()`, + and a behavioural one on the durable half of the fix (below). A one-shot shell check run + during implementation is not a regression test — the bug it guards is a *reordering*, which + is exactly the kind of edit a future refactor makes silently. +2. **Gate-change rebuild** — the directory refreshes when the gate signature changes, not only + on roster churn. Must be driven **through `_recompute_allowed_sender_ids` itself**, not by + calling the rebuild by hand: calling it by hand tests the predicate, which was never broken. + The cheap version is the isolation-disabled path, which needs no DB. +3. **Layer 2 truth table** — parametrised over star / mesh / gate-off: `idea_crosslab` drops in + star and stays in mesh; `pitch` the inverse. +4. **Layer 3** — `tagged_agent="pearce"` from `markham` is rejected, using the real production + JSON as the fixture. +5. **No-Slack-call assertion** — a rejected post must not call `post_message` and must not + increment `message_count`. This is what makes "reject" honest rather than cosmetic, and it + only earns that description if it drives **`_phase5_new_post` end to end** against a canned + LLM response. A test that calls the rejection helper directly and then asserts a stubbed + `_post_message` was not called proves nothing — the helper never calls it either way, so the + test passes just as happily when the call site was never wired up. + `tests/unit/test_simulation_logic.py:1166` (`TestPhase5ReplyActionSuppression`) is the working + pattern: stub `build_phase5_prompt`, monkeypatch `generate_agent_response`, `await + engine._phase5_new_post(agent)`, assert on `FakeSlackClient.posted`. +6. **`load_role` degradation** — one test per row in §5. +7. **Menu/enforcement agreement** — the rendered `{post_type_menu}` names exactly the + post-layer-2 set. +8. **Mesh behaviour** — `pi_lab` with gate `None` still offers every declared type and rejects + nothing. Layers 2 and 3 must be provably inert. +9. **Token pins** — `{post_type_menu}` must be added to the `leftover_tokens` list at + `tests/unit/test_roles.py:178` and the renderer-anchor list at `:370`. Without that, a + template carrying the token while the renderer does not substitute it would pass CI and leak + the raw `{post_type_menu}` into a live prompt. +10. **A new pi_lab phase-5 token/surgery pin.** Measured: `test_roles.py:160-227` pins the + scout_hub override only — the *global* `pi_lab` template's tokens and `funding_only` + surgeries are pinned nowhere. Since this change rewrites that template, add the equivalent + test for `pi_lab`. +11. **The default menu is role-aware.** `build_phase5_prompt(post_type_menu=None)` must render + the *calling agent's* role set, not `DEFAULT_POST_TYPES` unconditionally — otherwise a + `scout_hub` agent built by any direct caller is handed a menu offering `paper`, + `idea_crosslab` and `pitch`, none of which its `role.toml` allows. +12. **No enumeration is ever empty.** `render_menu` on a `targets` type with nothing to enumerate + must not emit `one of: .` — the case that otherwise reaches a committed snapshot. +13. **The reply-path funding bypass** — a blocked agent's `{"action": "reply", "post_type": + "funding_collab"}` to a non-funding thread is blocked. + +### Characterization snapshots will legitimately change — 8 of 9 + +Measured, not assumed. `agent-system.md` and `identity.md` are injected into **every** phase's +system prompt, so editing them moves almost every snapshot in +`tests/characterization/__snapshots__/test_agent_turn_gm.ambr`. Two more move because the drafts +now also edit `phase2-scan-filter.md` and `phase4-thread-reply.md`, whose bodies these tests +capture in `messages`, not just in `system`: + +| Snapshot | Why it changes | +|---|---| +| `test_scan_system_prompt_gm` | `Scripps Research`, `:test_tube:`, `:package:` (system only) | +| `test_system_prompt_public_vs_private_gm` | same | +| `test_thread_reply_system_prompt_gm` | same | +| `test_phase2_scan_prompt_flags_self_authored_gm` | the above **plus** the two new `phase2-scan-filter.md` exclusion rules | +| `test_phase4_prompt_phase_progression_gm` | the above **plus** the new `### If the other party is a scouting hub` section | +| `test_phase4_prompt_pi_context_and_funding_gm` | same as the row above | +| `test_reply_turn_composes_prompt_and_posts_gm` | same as the row above | +| `test_phase5_prompt_gm` | the system-prompt changes **plus** the whole Option C rewrite, the menu section, and the rendered default menu | + +Only `test_decide_phase_parses_scripted_json_gm` is unaffected. + +This does **not** license a blanket `pytest --snapshot-update`. CLAUDE.md's prohibition exists to +stop unintended drift being papered over, and it still binds. The rule for this change: + +- Regenerate those eight snapshots deliberately, then **read the diff line by line**. +- The diff must contain *only* text originating in the edited prompt files. +- The EXPLORE / DECIDE / CONCLUDE guidance strings from `src/agent/thread_guidance.py` must + appear **unchanged** in the diff. They are not touched by this work, and any movement in them + means something else broke. +- Baseline before starting: re-verified at `517a564`, after all five concurrent commits — `pytest + tests/characterization/test_agent_turn_gm.py tests/unit/test_roles.py + tests/unit/test_agent_prompts.py` gives **38 passed, 9 snapshots passed** (251s; the time is + testcontainers bringing up Postgres). Re-run it before touching a snapshot: a snapshot moving + for a reason that predates your change is the failure mode this baseline exists to rule out. + +### Prompt and code must land in the same change + +`prompts/` is bind-mounted into the agent container and re-read per call +(`docker inspect` confirms the mount; `agent.py:744` reads from disk with no cache), while +`src/` is **baked into the image**. So installing the prompt drafts without rebuilding the agent +image would put `{post_type_menu}` in front of a live renderer that cannot substitute it — the +raw token would reach real prompts. Ship the template edit and the renderer together, and +rebuild the agent image (`$DC --profile agent build agent`) before the next run. + +Gate: `./scripts/ci.sh`. No migration — `post_types` is config and `AgentRegistry.role` already +exists. + +## 7. Operational items + +These are one-off operations, not part of the code change. They are recorded here because the +cutover depends on their ordering. + +| Item | Method | Reversible | +|---|---|---| +| Disable `alanjary` (test-only bot) | `status='inactive'` in `AgentRegistry`; picked up live by `_sync_roster_from_db`, no restart | Yes | +| Delete its orphan cohort | `hub-alanjary` holds only `blackbird`+`grantbot`; the PI was never a member, so the agent is currently isolated to nobody | Yes | +| Clean working memory | Move `profiles/memory/*` to a timestamped backup — never `rm` | Yes | +| Delete mutilated Slack posts | **113 messages** (see below), via each authoring bot's own token | **No** | + +`--fresh` already wipes `agent_messages` (`src/agent/main.py:168`), so deleting the Slack copies +and then restarting `--fresh` leaves both sides consistent, with no orphaned DB rows breaking +the row-count-matches-Slack-message-count invariant documented at `_post_message`. `--fresh` +does **not** touch `profiles/memory/`, which is why the memory move is a separate step. + +**The deletion set, measured.** 200 top-level `:bulb:` posts carry the strip artifact; **113 of +them reached Slack** (the other 87 are DB-only, written while Slack was off or before tokens were +provisioned). They span 43 authoring agents and 6 channels. Only the 113 can be deleted — and +only the 113 need to be, since the rest never became visible. + +All measurements in this document come from a single simulation run, +`4f1e8395-8329-438d-99e8-d3bfeaa5ffb5` (started 2026-08-05 18:25 UTC, 671 messages). The agent +container was restarted at 21:50 UTC on 2026-08-06 and **resumed** that run rather than starting +a new one, so the counts remain current. + +The deletion set requires explicit sign-off before it runs. + +## 8. Out of scope, recorded + +Found during investigation, not addressed here: + +- **Working memory is not cohort-filtered** and names out-of-cohort labs on disk now (e.g. + `profiles/memory/epearce/public.md` lists six unreachable partners). Survives `--fresh`. + Mitigated for the next run by the memory move in §7, not fixed structurally. +- **`_prior_threads` is not cohort-filtered** (visibility only) and loads all `ThreadDecision` + rows across runs (`simulation.py:4060`). +- **`retrieve_profile` is not cohort-checked** and its path is unsanitised + (`src/agent/tools.py:233`). Verified reachable: `../private/blackbird` returns the hub's + private screening rubric; `../../CLAUDE` returns `CLAUDE.md`. Latent — 0 of 8 calls in the run + contained `..`. Wants a `resolve()`-and-check-parent guard. +- **`summarize_funding_thread(viewer_agent_id=…)` never reads that parameter** although both + callers pass it (`src/agent/funding_rules.py:172`, `simulation.py:1320`, `:2062`); its + `spinoffs` block scans the whole log (`funding_rules.py:211`). +- **Cohort strips are invisible at INFO.** Raising `simulation.py:2585` to INFO, or surfacing + the counter, is what makes the next occurrence of this class of bug visible in hours instead + of never. +- **Stale docstring path:** 21 citations across 9 files reference `.notes/cohort-system-v2.md`; + the file is at `specs/cohort-system-v2.md`. +- **`SLACK_ENABLED=true`** while the runbook's Phase 4 specifies DB-only for confidentiality, + which reopens runbook risk **A4** (the gate is behaviour, not access control). +- **Two Pearce labs:** `pearce` is Erika Pearce, `epearce` is Edward Pearce. diff --git a/docs/specs/2026-08-10-org1-parity-design.md b/docs/specs/2026-08-10-org1-parity-design.md new file mode 100644 index 0000000..53a2cea --- /dev/null +++ b/docs/specs/2026-08-10-org1-parity-design.md @@ -0,0 +1,645 @@ +# org1 parity: porting the generic blackbird work to copi.science + +**Date:** 2026-08-10 +**Status:** Design, approved. Not implemented. +**Branch:** `org1-parity` (off `origin/cohort-db-conversations` @ `5fa6219`) +**Destination:** `main`, via `cohort-db-conversations` + +## 1. The problem + +Five branches carry production-relevant work, and no two of them agree: + +| Branch | Head | Relationship | +|---|---|---| +| `main` | `b7edcbc` | ancestor of `cohort-db-conversations` | +| `cohort-db-conversations` | `5fa6219` | `main` + 77; **strict ancestor of `blackbird`** | +| `blackbird` | `22dd952` | `cohort-db-conversations` + 119 | +| `copi-prod` | `0c7c4be` | **what org1 actually runs.** Not an ancestor of anything above: 4 unique commits, alembic head `0018` | +| `blackbird-main` | `ac5a2c9` | deprecated | + +Because `cohort-db-conversations` is a strict ancestor of `blackbird` +(`git merge-base` returns cohort's own head; `rev-list --left-right` is `119 / 0`), +reconciliation is **subtraction, not merge**: decide which of 119 commits belong on +org1. `copi-prod` is the exception — it is genuinely divergent and must be merged in, +or a deploy silently reverts production. + +### 1.1 The two instances + +| | **org1** | **blackbird** | +|---|---|---| +| Path / compose project | `/home/ubuntu/copi-python`, `copi-python` | `/home/ubuntu/blackbird-copi-science`, `copi-blackbird` | +| Domain | `copi.science` | `blackbird.copi.science` (proxied **by org1's nginx**) | +| Web service | `app` | `blackbird-app` (an *uncommitted* host-local compose edit; the tracked `docker-compose.prod.yml` says `app` on every branch) | +| Roster | Scripps Research labs | 56 agents, 57 of 60 profiles Johns Hopkins | +| Topology | **mesh** — peer-to-peer lab collaboration | **star** — 56 cohorts of `{pi, blackbird, grantbot}` | +| Cohort gate | `cohort_isolation_enabled=False` | `True`, `cohort_default_policy="isolated"` | +| Product | cross-lab collaboration proposals | BlackbirdBot screens PI ideas for patentability / fundability / commercialisability | +| alembic | **0018** | 0025 | + +Sources: `CLAUDE.md:56-79` (blackbird), `docs/specs/2026-08-06-role-topology-post-type-gating-design.md:9-15,69,145`, +`specs/cohort-system-v2.md:390-405`, `src/config.py:347-370`, commit `0e1ac52`'s message, +`copi-prod:476f46b`. + +The divergence is topological, and the design doc for post-type gating states it +plainly: *"`pi_lab` in org1's mesh **should** make cross-lab idea posts — that is the +product. The same role in this star must not."* + +## 2. Decisions + +1. **Scope: generic parity, minus the Blackbird product.** Everything that is not the + scouting product: cohort feed + topology fixes, security/500 fixes, the lint + + hermeticity repair, the rate limiter, role infrastructure, post-type modules. +2. **org1's cohort gate is off now, but cohorts are coming.** So the feed gating lands + *before* the flip, and post-type enforcement waits *for* it. +3. **`copi-prod` is merged in first**, establishing the invariant that the branch is + never behind production. +4. **org1's prompts stay frozen.** No base-prompt commit is ported. The post-type + machinery lands inert; enforcement is deferred. + +## 3. Approach + +Ordered cherry-pick in blackbird chronological order, with surgical commits +hand-applied. Chosen over (a) merging blackbird whole and reverting the product — +which would put migration 0025 on org1's branch and require restoring the `.ambr` +snapshot — and (b) feature-squashing to end state, which discards commit messages +that in this repo carry the measured evidence ("146 of 146 tagged posts addressed an +unreachable agent", "265 → 257 findings", "took the hub off the air for 161 turns"). + +Measured conflict cost of ordered replay: **9 commits / 13 files** of 54. Every +hand-applied commit carries a `Ported-from: (partial)` trailer naming what was +dropped, so the original reasoning stays findable on `origin/blackbird`. + +## 4. The port set + +**53 commits ported in full, 9 in part, 57 excluded entirely. Branch: 64 commits.** + +Ported in part: `9714f26`, `6b76f27`, `21869e2`, `29fc8f1`, `f32a83e`, `e116feb`, +`1b44e1c`, `0a57e41`, `10d598f`. Each contributes one or more generic hunks to a +hand-applied commit; the rest of each is dropped. + +`21869e2` is hand-applied rather than cherry-picked: its hunk anchors to a +`_strip_assessment_sidecar(text)` call this branch does not have, its comment has to be +reworded around that absence, and the return contract it needs comes from a different +commit (`29fc8f1`). One fresh commit citing both is cleaner than a pick plus three +in-flight edits. + +### Phase 0 — `git merge origin/copi-prod` (1 commit) + +Verified by `git merge-tree`: four conflicts, **zero lines of code**. + +| Conflict | Resolution | +|---|---| +| `.gitignore` | union — cohort's `docs/superpowers/` plus copi-prod's `.claude/`, `logs/`, `scripts/_*`, `*.bak.*` | +| `scripts/audit_pub_dois.py` | take both: copi-prod redacted a **real ORCID** (`0000-0002-9943-7557` → placeholder) in the usage block; cohort changed the invocation to `docker compose exec app`. The redaction is a privacy fix in a public repo and must not be lost. | +| `scripts/backfill_agents.py` | take both (copi-prod moved seed files under `data/cohorts/`; cohort changed the invocation) | +| `scripts/generate_sparsedata_user.py` | take both, same shape | + +`src/config.py` auto-merges, so `audit_recipients` / `audit_recipient_list` arrive +free. `.dockerignore`, the `copi-edge` external network, the `blackbird.copi.science` +vhost, and 9 maintenance scripts arrive as fast-forward adds. + +**What this prevents.** Deploying cohort as-is to org1 would have: deleted the tracked +`.dockerignore`, so `Dockerfile`'s `COPY . .` resumes **baking `.env` secrets into +image layers**; reverted the `copi-edge` network and vhost, taking +**blackbird.copi.science dark** (org1's nginx is the edge for both instances); +reverted `audit_recipients` to hardcoded addresses; and deleted 9 operator scripts. + +**Post-merge assertion.** `git diff org1-parity origin/copi-prod` must show only +removals cohort made deliberately (the dead onboarding templates from `d1005b1`, the +pre-0019 alembic files). Anything else is a hidden prod revert. + +### Phase 1 — Buy lint headroom (1 hand-authored commit) + +`cohort` sits at **260 `src/` ruff findings against `ci.sh`'s 260 ceiling** — zero +headroom, so the first feature commit breaches it. + +- Cherry-pick verbatim from `3a23e73`: `src/agent/message_log.py`, + `src/dependencies.py`, `tests/integration/test_proposal_review.py`, + `tests/unit/test_email_templates.py`, `tests/unit/test_slack_tokens.py` — all five + blob-identical to `3a23e73^`. +- Hand-apply three one-line import removals: `typing.Any` and `PostRef` from + `src/agent/agent.py`, `sys` from `src/agent/main.py`. (These conflict as patches + because blackbird's parent has the role imports, but the removals themselves apply — + cohort's per-file counts of 3 and 4 match `3a23e73^` exactly.) + +Effect: `src/` 260 → ~253. `Ported-from: 3a23e73 (partial)`. + +### Phase 2 — Role infrastructure (9 picks + 2 hand-applied) + +``` +a655ede feat(roles): prompt-path resolution with per-role fallback +46a8391 feat(roles): role.toml manifest with tool allow-list and safe fallbacks +ac2da9e refactor(agent): role-aware prompt loading; collapse 3 builders into 1 +48e4d05 feat(db): add agents.role column (migration 0024) +ffef698 fix(migrate): advance migration tooling's target from 0023 to 0024 +da0625d feat(roster): thread role through roster reads; pick up role changes live +711b13b feat(tools): per-role tool allow-list, enforced in Phase 4 and the executor +bc293d9 fix(cohort): scope lab directory to the cohort gate (runbook A3) +4ec8ab7 feat(admin): view and set agent role; show role on topology page +``` + +Verified no-ops for `pi_lab`: `resolve_prompt_path` falls through to +`prompts/{file}`; `roles.DEFAULT_TOOLS` is **exactly** cohort's four +`TOOL_DEFINITIONS` entries; `ac2da9e`'s `prompts/identity.md` is byte-identical to the +three duplicated literals, **"at Scripps Research" included**, and moved zero +snapshots. `711b13b` lands before `5367027` in blackbird order, so it applies clean and +yields `tools_for_role` with only the four base tools. + +**Hand-applied A — `6b76f27` (partial), 2 lines.** `build_phase5_prompt` loaded +`PROMPTS_DIR / "phase5-new-post.md"` directly instead of `_load_prompt()`. Without +this the role mechanism is internally inconsistent: agent-system, identity, phase-2, +phase-2-prune and phase-4 honour role overrides; phase 5 silently does not. pi_lab's +phase-5 snapshot is unchanged (stated in the commit). Drop everything else in +`6b76f27` — it is the scout_hub prompt tree. + +**Hand-applied B — migration-tooling completion.** Four edits `ffef698` does not carry: + +1. `tests/integration/test_harness_smoke.py`: head pin `0023` → `0024` + (`Ported-from: 9714f26 (partial)` — the rest of that commit is the live + PatentsView test). +2. `scripts/migrate/preflight.py`: add `PlannedObject("0024", "column", "role", "agents")`. + The entry lives in `517a564` (excluded), so without it `REVISION_ORDER` reaches + 0024 while preflight's collision check silently skips `agents.role`. +3. `tests/unit/test_migration_checks.py`: extend the drift-guard tuple from + `("0019"…"0023")` to include `"0024"`. +4. `scripts/migrate/postflight.py`: add + `("agents", "role", "character varying", False, "'pi_lab'::character varying")` to + `EXPECTED_COLUMNS`, so 0024 is *verified* after the window rather than assumed. + Blackbird documented this gap (`VERIFIED_REVISIONS`); closing it is cheap here + because 0024 creates no table, so `CHAIN_CREATED_TABLES` is unaffected. + +Also reword `ffef698`'s comment: it justifies adding `0023` to +`SUPPORTED_START_REVISIONS` as "production's current stamp", true for blackbird and +false for org1 (0018). + +### Phase 3 — Cohort feed and topology (18 picks) + +``` +8bc0e24 docs: design for cohort-scoped conversations feed, threads, topology payload +d84ce6b docs: implementation plan for cohort-scoped feed, threads, topology payload +0efd6a5 fix(admin): topology matrix payload 3,360 fields -> 116 +d2b3b21 fix(admin): bound the topology cross product by table size, not payload size +ffad1a1 feat(feed): gate_clause — the cohort gate as a SQL predicate +942b31b feat(feed): resolve_agent_gate via the engine's compute_gates +5bf587e docs(feed): clarify resolve_agent_gate docstrings post-review +230a4c0 fix(feed): cohort-scope the conversations page and select thread roots +6d94148 fix(feed): own-post carve-out, gate the reply count, and pin the regressions +f77a0a2 feat(feed): thread expand endpoint returning a gated replies partial +ddb3892 fix(feed): prove replies are actually gated; dedupe channel-set computation +0c04be6 feat(feed): render roots with a reply badge and expand-on-click +0dd94db fix(feed): guard the thread-expand link against a double-click mid-fetch +44f1ad0 fix(feed): cover the plural badge and href correctness gaps from review +cc3c90f docs(cohort): correct spec/docstrings now that the PI feed is gated +4bc5cbe fix(feed): scope reply queries to the root's channel; log preflight fail-open +a968d7a test(admin): pin the topology marker/cell cross-product invariant +7f6b304 docs(cohort): fix the amendment pointer to specs/, not .notes/ +``` + +Two of these are live defects on org1 today, independent of the gate: + +- **The admin topology matrix cannot be saved.** 60×56 posts 3,528 form fields against + Starlette's `max_fields=1000`. `d2b3b21` additionally bounds a multiplicative + cross-product DoS (25k ids per side = 50k fields, under the cap, a 625M-entry set). +- **The PI conversations feed had no content filter** beyond channel name. Inert while + the gate is off (`gate_clause(None)` returns `true()`), but it is the fix that makes + the gate flip safe, and `specs/cohort-system-v2.md` §6.2 is amended to record the + deliberate narrowing of "the gate is not access control". + +`0efd6a5` touches `templates/admin/cohort_topology.html`, which also carries the Role +column from `4ec8ab7` — Phase 2 lands first, so it applies clean. + +### Phase 4 — Scheduler and rate limiter (15 picks) + +``` +15a277e docs(spec): load-proportional budget and scheduling for star topologies +7c6768e docs(plan): implementation plan for load-proportional budget/scheduling +5654271 docs(plan): adversarial audit against HEAD 7f6b304 — fix 4 defects +09b83aa feat(sched): _agent_load — the shared load signal +9932645 feat(config): rate-limiter settings + optional per-role allowance +0929870 feat(sched): call ledger — record_api_call maintains both counters +6d1deed fix(roster): adopt a Slack client when a live agent gains a token +0821372 feat(sched): sliding-window rate limiter replaces the cumulative cap +92e4989 feat(sched): rebuild call_times from llm_call_logs within the window +f5531d2 fix(sched): make step 4b idempotent and DB-test the window query +e111732 feat(sched): load-proportional selection weight and reactive tiebreak +bc4dd3a feat(cli): deprecate --budget, default it off, document the replacement +00e174f test(sched): production regression for the run-4f1e8395 hub bench +3a23e73 fix(ci): make settings-dependent tests hermetic; clear lint debt +46d3a61 fix(sched): a throttled roster must back off, not end the run +``` + +Ported as one unit — `Agent.record_api_call` is the single write point for both +counters, and the `pi_handler.py` / `llm.py` `on_retry` hooks are what keep the live +and rebuilt ledgers consistent. The bug it fixes is generic and severe: `_rebuild_state` +restores `api_call_count` from `llm_call_logs`, so a crossed cumulative cap benches an +agent **permanently, across restarts**. + +Conflict resolutions in this phase: + +- `9932645` / `src/config.py` (blocked by `0621ef3`): keep only + `llm_rate_window_seconds`, `llm_calls_per_load_per_window` and + `_guard_rate_limiter_settings`. Drop the `uspto_api_key` / `patentsview_api_key` + block. +- `9932645` / `tests/unit/test_roles.py` (blocked by `6b76f27`): see the + `test_roles.py` trim at the end of this phase. +- `0929870` / `src/agent/agent.py` (blocked by `6b76f27`): take only the + `record_api_call` hunk. +- `bc4dd3a` / `CLAUDE.md` (blocked by `996cca7`): drop the hunk. org1's CLAUDE.md is + authored separately (§8). +- `3a23e73` / `tests/unit/test_patents.py` (blocked by `4322e5c`): drop the hunk. + Its `src/agent/agent.py` and `src/agent/main.py` hunks are already in Phase 1; take + the remaining test hermeticity fixes for `test_agent_page.py`, `test_cohort_admin.py` + and `test_roles.py`. + +**`test_roles.py` — nothing is deleted; the scout_hub tests never arrive.** Blackbird's +copy ends at 27 tests, of which 10 read the real `prompts/roles/scout_hub/` tree. Those +10 are *introduced* by excluded commits (`6b76f27`, `2bd0289`, `9d5a1d7`, `3f1f91d`, +`eadde02`, `988eac1`, `5114905`, `f7a9f68`), so on this branch they are never written in +the first place. Keeping them would have dragged `patents.py`, `blackbird_rubric.py` and +`specialists.py` along — they assert `"search_prior_art" in spec.tools`, quote the +Blackbird rubric, and check the Baltimore gating criterion. + +What that means operationally is *conflict resolution, not deletion*. Three ported +commits patch this file and each needs a specific decision: + +- `9932645` adds five `tmp_path`-based rate-override tests. **Keep all five.** The + conflict is purely positional — they append after scout_hub tests that do not exist. +- `dc371af` adds six tests. **Keep the four `tmp_path`-based `post_types` tests; drop + `test_scout_hub_declares_its_two_post_types` and + `test_scout_hub_cannot_post_a_cross_lab_idea`**, which call + `load_role("scout_hub")` and need the `role.toml` this branch does not ship. +- `3a23e73`'s hunk edits the `from pathlib import Path` and `_load_role_real` import + lines. **Drop the hunk entirely** — both imports exist only to serve the scout_hub + tests, so neither is present here and there is nothing to lint. + +Final content: the mechanism tests from `a655ede` and `46a8391`, five rate-override +tests from `9932645`, four `post_types` tests from `dc371af`. Every one uses `tmp_path` +and a monkeypatched roles directory; none touches `prompts/`. + +### Phase 5 — Role prompt completion and generic fixes (6 picks + 4 hand-applied) + +``` +2467229 fix(agent): phases 2 and 4 must honour role prompt overrides +bc40d20 fix(agent): phase2-prune must also honour role prompt overrides +683c09a feat(scout_hub): drive the interview off the screening rubric [thread_guidance extraction] +44f09be fix(llm): detect and log a still-truncated retry; let callers count it +73a78c3 fix(admin): a Slack post with no mappable sender must not 500 /admin/discussions +5fb68c0 fix(admin,public): close the null-agent_id 500 class, an unauthenticated vote-tamper hole +``` + +Plus four hand-applied: the `_post_message` suppression and `-> bool` contract +(`21869e2` + `29fc8f1`), and hand-applied C, D and E below. + +`683c09a` extracts `thread_guidance.py`; its `_PI_LAB` strings are byte-identical to +the pre-refactor `agent.py` literals and are pinned by the snapshot. Its `_SCOUT_HUB` +dict is dead code without the role, and is kept rather than trimmed so the file stays +mergeable with blackbird. + +`5fb68c0` closes an **unauthenticated vote-tamper hole**: `if vote_obj.voter_token and +token and ...` meant omitting `voter_token` set `token = None` and bypassed the +ownership check entirely. + +Conflict resolutions: + +- `73a78c3` / `src/routers/admin.py` (blocked by `f32a83e`): hand-apply the + `available_agents` None-guard. +- `5fb68c0` / `src/agent/simulation.py` (blocked by `10d598f`) and + `tests/integration/test_opportunity_assessment_persistence.py` (blocked by + `66948dc`): **drop both.** The `simulation.py` hunk is the assessment-persist fix; + the test file is Blackbird product. Keep `public.py`, `admin.py`, the two templates, + and the two characterization tests. +- `21869e2` / `src/agent/simulation.py` and `tests/unit/test_simulation_logic.py` + (blocked by `f4488f7`, `1462d29`): hand-apply the empty-post suppression onto + cohort's `_post_message`. Three adjustments, none of them mechanical: + + 1. **The return contract must come with it, as `-> bool`.** `21869e2` alone writes a + bare `return`, which is useless to `e116feb`'s caller guards below. On blackbird + the contract arrives in `29fc8f1` (`-> None` → `-> bool`, `return False` at both + bail-outs) and is then widened to `-> str | None` by `1b44e1c` — but that widening + exists *only* so an `opportunity_assessments` row can store the post's `slack_ts`, + which org1 has no use for. Take `29fc8f1`'s `-> bool` and stop there. + `Ported-from: 21869e2, 29fc8f1 (partial)`. + 2. **Reword the comment.** `21869e2`'s hunk sits directly below + `text = _strip_assessment_sidecar(text)` and its comment is written around that + call, which does not exist on this branch. The guard is still correct — a truncated + response can strip to empty through the `` substitution alone — + but the rationale must be restated in those terms rather than inherited. + 3. Insert after `text = re.sub(r"", "", text).strip()`, which is the + line the hunk actually anchors to once the sidecar call is gone. + +**Hand-applied C — `f32a83e` (partial).** `generate_with_tools` gains `on_retry`, both +of its retry sites re-check `stop_reason`, and `simulation.py`'s phase-4 call site +passes `on_retry=agent.record_api_call`. `generate_with_tools` is **the function +phase-4 thread replies use** — org1's entire product — and `44f09be` fixes only +`generate_agent_response`. Without this, one retry site swallows truncation silently +and the limiter undercounts every retried phase-4 turn. Drop B1/B2/B3 (`admin.py` +triage scoping, `assessments.html`). + +**Hand-applied D — `e116feb` (partial).** The three `_post_message` caller guards: +the phase-4 reply site, the phase-5 private-channel flat follow-up, and the phase-5 +thread-creating reply. `21869e2` makes `_post_message` return falsy; this is what makes +the callers *check* it. Without it, half the callers count the turn, clear +pending-reply/backoff state, and move posts into `active_threads` for a message nobody +saw. Drop the `_extract_assessment_json` rework and the assessment logging triage. + +**Hand-applied E — `1b44e1c` (partial), 12 lines.** `action = action_data.get("action")` +plus a return when falsy. Cohort defaults a missing `action` to `"new_post"` and posts +anyway. **Drop the `max_tokens` 1000 → 2500 change in the same commit** (§7). + +### Phase 6 — Post-type machinery, inert (5 picks + 2 hand-applied) + +``` +3fd8a91 fix(cohort): build the lab directory after the gate, not before +f231bc8 feat(post_types): the canonical vocabulary and the role+topology filter +20065e1 feat(post_types): add legacy idea->idea_crosslab alias resolution +dc371af feat(roles): parse a post_types allow-list from role.toml +f2cbfe9 feat(agent): substitute {post_type_menu} in the phase-5 prompt +``` + +**No enforcement call.** `66948dc` is excluded (§7). The machinery is inert on org1: +`f2cbfe9` substitutes via `str.replace`, and org1's `prompts/phase5-new-post.md` has +no `{post_type_menu}` token, so nothing renders and nothing is judged. + +`3fd8a91` is a no-op on a mesh (the directory filter only bites with the gate on) and +is kept as future-proofing for the flip. + +Conflict resolutions: + +- `3fd8a91` / `src/agent/simulation.py` (blocked by `f32a83e`): hand-apply the + directory-after-gate reordering. +- `dc371af` / `prompts/roles/scout_hub/role.toml` (blocked by `6b76f27`): drop — the + file is not on this branch. `dc371af` / `tests/unit/test_roles.py` (blocked by + `f7a9f68`): per the Phase 4 trim. + +**Hand-applied F — `0a57e41` (partial).** `parse_post_types` dedupes by name (dict, +last-wins, first-occurrence order preserved) with a WARNING. A real parse bug in code +we are porting: duplicate `[[post_types]]` entries produced two contradictory entries +while lookup kept only the last. The `render_menu` wording hunk in the same commit is +inert here (no menu is rendered) but is taken for file fidelity. **Drop the +`simulation.py` body-mention rejection and skip-backoff hunks** — both exist only to +serve enforcement. + +**Hand-applied G — `10d598f` (partial).** `_recompute_allowed_sender_ids` refreshes lab +directories **even when the membership query raises**, so a stale-but-correct gate does +not leave a directory absent rather than merely stale. Pairs with `3fd8a91`, which must +land first. Plus two comment corrections in `post_types.py`. **Drop** the +`_normalize_tagged_agent` work, the `_post_type_rejections` counter, the +`_cohort_gate_banner.html` hunk, and every prompt hunk. + +### Phase 7 — Post-type design doc (1 hand-authored commit) + +Land `docs/specs/2026-08-06-role-topology-post-type-gating-design.md` at its final +state — the content of `d6bf5d7` as amended by `a187a1d`, `31cb20c` and `454fa86` — +so `post_types.py`'s docstring citation resolves. **Without** +`docs/specs/2026-08-06-post-type-gating-prompts-draft/`, which is blackbird's prompt +tree including scout_hub. + +## 5. Migration and deploy + +**Target: `0018 → 0024` in one window.** The runbook already exists on this branch +(`docs/production-migration.md`, 596 lines) and supports 0018 as a start revision; +its title and target move `0023` → `0024` and 0024 is appended to its §1 step table. + +`0018 → 0019` is the expensive step: 7 columns, 4 indexes and +`uq_agent_messages_run_ts` on `agent_messages` under `ACCESS EXCLUSIVE`. Treat it as a +**full outage on that table** — a queued `ACCESS EXCLUSIVE` request blocks arriving +readers, not just writers. It hard-fails if duplicate `(simulation_run_id, message_ts)` +rows exist. + +1. Read-only measurement (runbook §2, Q1–Q4): table size, duplicate groups, blocking + sessions. Sizes the window before anything is touched. +2. `scripts/migrate/remediate_duplicates.py` dry run (executes in a `READ ONLY` + transaction), then `--apply` if needed. +3. Rehearse: `scripts/migrate/preflight.py`. Writes nothing. +4. Apply: `COMPOSE_FILE=docker-compose.prod.yml ./scripts/migrate/run_migration.sh --apply`. + Takes its own `pg_dump -Fc` and verifies the TOC first. +5. `scripts/migrate/postflight.py` against the step-1 snapshot. + +`run_migration.sh` defaults `SVC=app`, which is correct for org1 (unlike blackbird). +But it shells out to **bare `docker compose`**, which resolves `docker-compose.yml` — +the dev stack. `COMPOSE_FILE=docker-compose.prod.yml` is not optional. + +**Order: migrate → rebuild → restart.** Nothing migrates automatically: the prod web +command is a bare `uvicorn` and there is no `create_all`. New code against 0018 fails +at roster load, because `src/agent/main.py` selects `AgentRegistry.role`. The `agent` +service bakes `src/` into its image, so it needs `--profile agent build agent`, not +just `up -d --build app worker`. + +**Rollback, and when the door closes.** `0024`'s downgrade is `if_exists`-guarded and +safe. `0019`'s is not: it runs +`alter_column("agent_messages", "agent_id", nullable=False)`. Immediately after the +window there are no NULLs, so a downgrade works — but as soon as the new code runs, +`_rebuild_state_from_slack` writes `is_bot=True, agent_id=NULL` for any Slack sender it +cannot map to a known bot (7 such rows measured on blackbird). **The moment the first +one lands, `alembic downgrade` past 0019 stops being an option and the only rollback is +restoring the step-4 dump.** Decide in advance how long that door is held open. + +## 6. Verification + +`./scripts/ci.sh` is the whole gate: alembic single-head and no duplicate ids → +upgrade→downgrade→upgrade round trip on a throwaway Postgres → ruff on `tests/` at zero +→ ruff ratchet on `src/` at ≤260 → full pytest with branch coverage ≥60. + +**The decisive check — snapshot invariance.** The branch's central premise is that +org1's agent behaviour does not change. That reduces to one assertion: + +``` +git diff origin/cohort-db-conversations -- tests/characterization/__snapshots__/ # must be EMPTY +``` + +The only three commits in blackbird's 119 that touch `test_agent_turn_gm.ambr` are +`0e1ac52`, `0a57e41` and `10d598f` — all excluded, the latter two only in part, and +neither part touches the snapshot. A non-empty diff at any checkpoint means a prompt +changed and the port is wrong. **Never run `pytest --snapshot-update` to reconcile it.** + +**The lint ratchet.** cohort is at 260 findings against a ceiling of 260. Phase 1 takes +it to ~253; the feature work adds ~+5 (`agent_page.py` +2, `admin.py` +2, +`conversation_feed.py` +1); projected final ~258. Blackbird's own head is 259 +*including* the assessments route and four excluded modules, so ~258 is consistent from +both directions. **Measured, not assumed, at every checkpoint.** If it lands over 260 +the fix is paying down debt in the files we touched — `SRC_LINT_MAX` is not raised. + +**The institution guard arrives correct — do not edit it.** Blackbird's +`tests/unit/test_agent_prompts.py:17` reads `assert 'Scripps Research' not in prompt`, +which would fail here. But that line is written by the **excluded** `0e1ac52`. The +version `ac2da9e` creates — the one this branch gets — asserts +`'the Andrew Su lab at Scripps Research' in prompt`, and no other ported commit +(`2467229`, `bc40d20`, `f2cbfe9`) touches it. So it lands as a guard that a future port +cannot silently de-institutionalise org1's prompts, and it is *stronger* than an +inverted `'Scripps Research' in prompt` because it pins the whole phrase. + +The related assertion `"{post_type_menu}" not in messages[0]["content"]` (from +`f2cbfe9`) also passes unchanged: org1's `prompts/phase5-new-post.md` carries no such +token, so there is nothing to leave unsubstituted. + +**Checkpoints.** `ci.sh` takes ~6 minutes, so it runs after Phase 1 (proves headroom +bought), after Phase 3 (largest surface), after Phase 4 (largest behavioural risk), and +at the end. Between checkpoints: `ruff check src --quiet | wc -l`, the snapshot diff, +and `alembic heads`. + +**What the round trip cannot prove.** With 0024 in the chain the round trip runs +`upgrade head → downgrade 0018 → upgrade head` against an **empty** throwaway database. +It will pass, and it cannot catch 0019's `agent_id → NOT NULL` downgrade failure, which +only occurs when rows exist. §5's rollback note is the mitigation. + +**Coverage.** `COV_MIN=60`. Each feature is ported *with* its tests +(`test_conversation_feed.py` 926 lines, `test_hub_budget_scheduler.py` 730, +`test_post_types.py` 420, `test_roles.py` minus the 10 scout_hub tests, +`test_llm_service.py`, `test_thread_guidance.py`, `test_tool_gating.py`, +`test_lab_directory_ordering.py`, `test_state_rebuild.py`), so coverage should rise. +Porting `src/` without its tests is the one way this floor breaks. + +**Hermeticity.** `3a23e73`'s test fixes matter only on a host with a provisioned `.env`: +7 tests read `SLACK_ENABLED`, `COHORT_ISOLATION_ENABLED` and +`OUTBOUND_EMAIL_ALLOWLIST` from it. The development checkout sets none of the four, so +the gate is green there today; a prod host would see those 7 fail before the port +begins. + +## 7. Exclusions + +**57 commits excluded entirely.** Every hash below is dropped in full; the 8 partials +(§4) are deliberately absent from this table. + +| Group | Count | Commits | +|---|---|---| +| Patents / USPTO prior-art | 13 | `ce30c5f 27e88cd 5367027 5deac1e 0621ef3 4322e5c 9d4afc9 f32b7fe 678dfd1 1868089 b034e31 506d763 517a564` | +| Blackbird rubric, `opportunity_assessments`, triage UI | 12 | `3b59bd3 fcb6e7b c6943d4 53d4410 e91e6c5 1462d29 792f153 f4488f7 3919acc 00d5ebd a247ed8 265cd48` | +| Nine-evaluator specialist panel | 7 | `ebe03b0 3f3b992 a64b0ff d99656b ccd6f22 2e68d64 c9298fb` | +| scout_hub prompt content | 9 | `61dc019 2bd0289 9d5a1d7 3f1f91d eadde02 988eac1 5114905 f7a9f68 2af98de` | +| Base prompts — frozen | 3 | `0e1ac52 6fa3980 22dd952` | +| CLAUDE.md / instance runbooks | 3 | `996cca7 52f9e9a 805b6bd` | +| Blackbird design and plan docs | 8 | `6b7e7e7 87e1670 1f30556 663ea33 d6bf5d7 a187a1d 31cb20c 454fa86` | +| Deliberate deferrals | 2 | `66948dc 96c6243` | + +`6b76f27` and `29fc8f1` are *not* listed here — each contributes one generic hunk (the +phase-5 `_load_prompt` fix; the `-> bool` return contract) and is otherwise dropped. +`53 + 9 + 57 = 119`. + +`0e1ac52` is the one never to take: it strips "at Scripps Research" from +`agent-system.md`, `identity.md` and `_DEFAULT_IDENTITY` because, in its own words, +"57 of 60 public profiles say Johns Hopkins". Correct for blackbird, wrong for org1. + +`d6bf5d7` and `454fa86` also carry +`docs/specs/2026-08-06-post-type-gating-prompts-draft/`, i.e. blackbird's full prompt +tree including scout_hub. Phase 7 lands the design doc only. + +### 7.1 The two deliberate deferrals + +**`66948dc` — post-type enforcement.** Layers 2 and 3 are provably dormant with the +gate off (`simulation.py` returns early when `allowed_sender_ids is None`), but +**layer 1 is not**: a model that omits or invents a `post_type` would publish nothing +where it publishes something today, against a prompt that never states a vocabulary is +enforced. org1's current enum +(`introduction|paper|help_wanted|idea|idea_crosslab|funding_collab`) is fully covered by +`DEFAULT_POST_TYPES` plus the `idea` alias, so the risk is small — but the benefit in a +mesh is near zero, because the artifact enforcement exists to prevent (259 `:bulb:` +posts, 0.8% reply rate, 146/146 tagged posts unreachable) is a star-topology pathology. +Enable it when cohorts flip on, together with a purpose-built org1 prompt variant, as +its own change with its own before/after measurement. + +**`96c6243` — terminal-artifact backpressure.** Its `nothing_postable` condition +removes a cost-saving early return **for every role**. A blocked mesh `pi_lab` agent +always has `funding_collab` nominally available, so the early return stops firing and +the agent burns an LLM call to reach a phase-5 turn it then skips for lack of an FOA. +Pure cost on org1; the hub was the only beneficiary. Also drops `TERMINAL_POST_TYPES`, +which nothing else on this branch references. + +### 7.2 Dropped from inside a ported commit + +`1b44e1c`'s `max_tokens` 1000 → 2500 on the phase-5 call is **unconditional across all +roles**, sized for scout_hub's 11-section assessment artifact plus its JSON sidecar. On +org1 it is a 2.5× output-token ceiling increase with nothing to spend it on. Phase 5 +takes only that commit's `action` guard. + +## 8. Out of scope, recorded + +- **org1's `CLAUDE.md` is authored separately, not ported.** Blackbird's version + documents `blackbird-app`, `blackbird-agent-run`, the `copi-edge` two-stack warning + and a `docker-compose.prod.yml` service name that **does not exist in the tracked + file on any branch**. Its *Testing* section, however, is a doc-accuracy fix that + applies to both instances: cohort's `CLAUDE.md` still describes the in-container + pytest path and omits the round trip and `src/` ratchet that cohort's own `cc8490f` + added. Salvage that section; discard every service name. +- **`coPI-podcast`** carries 66 unmerged commits (podcast/TTS, PI proposal evaluations, + focus-agent mode). Untouched here; it is a separate reconciliation. +- **`blackbird-main`** is deprecated and has one unique merge commit. + +### 8.1 A dangling citation, recorded rather than fixed + +`docs/specs/2026-08-05-hub-bot-customization-design.md` is cited by `src/agent/roles.py`'s +module docstring, by migration `0024`'s docstring, and by +`docs/specs/2026-08-06-role-topology-post-type-gating-design.md` (which quotes it at +`:261` for the claim that runbook gap A3 was "recorded as closed"). **It exists on no +branch and was never committed.** The role mechanism's actual design is `roles.py`'s +module docstring plus migration `0024`. + +Deliberately not fixed in code. A stub cannot satisfy a line-number citation, and +editing the three citations would diverge `roles.py` and — worse — migration `0024`'s +docstring from blackbird's copies. Migration files are what people diff across +deployments when debugging a schema mismatch, and a comment-only delta there is noise +that reads as signal. It would also guarantee a conflict on the next port in either +direction. Recording it here costs nothing and puts the answer where a reader chasing +the citation should end up. + +The org1-specific role facts that would otherwise have lived in that document: + +- The role mechanism ships, but **only `pi_lab` is ever assigned** on org1. No + `AgentRegistry` row is set to `scout_hub`, and `prompts/roles/scout_hub/` is + deliberately absent, not missing. +- `roles.DEFAULT_TOOLS` is exactly the four base tools + (`retrieve_profile`, `retrieve_abstract`, `retrieve_full_text`, `retrieve_foa`), which + is precisely cohort's `TOOL_DEFINITIONS`. The allow-list therefore grants and removes + nothing. It is explicit rather than "every tool" so that a newly added tool stays + opt-in. +- `prompts/identity.md` keeps "at Scripps Research". This is a deliberate divergence + from blackbird and is guarded by `tests/unit/test_agent_prompts.py` (§6). + +## 9. Open items + +1. **How long the 0019 rollback door stays open** (§5). A judgement call about how long + to hold the pre-window dump as the viable rollback before the first NULL + `agent_id` row makes `alembic downgrade` unusable. +2. **Whether to enable post-type enforcement when cohorts flip on**, and whether that + change authors an org1 prompt variant or keeps prompts frozen and accepts a menu + that never renders (§7.1). +3. **`copi-prod`'s future.** After this branch merges to `main`, `copi-prod` should + either be deleted or reduced to a deploy tag, so a fifth line does not re-accumulate. + +## 10. Audit corrections (2026-08-10, pre-execution rehearsal) + +A full adversarial rehearsal — the merge, every pick, every hand-edit, ending in a +green `./scripts/ci.sh` (single head 0024, clean round trip, tests/ lint 0, src/ +256/260, 1654 passed, coverage 64.66%, 20 snapshots) — corrected the following +claims in this document. The plan carries the executable versions. + +- **§4 partition.** `f2cbfe9` joins the ported-in-part set: four of its tests assert + the phase-5 menu *renders*, which needs the `{post_type_menu}` token only the + excluded `0e1ac52` adds. Final split: **52 in full, 10 in part, 57 excluded.** + (§7's "the 8 partials" undercounted even the original nine.) +- **§4 conflict map.** `9932645` conflicts only in `test_roles.py` — `0621ef3`'s + USPTO block never landed here, so `src/config.py` merges clean and there is + nothing to delete. `0929870`, `73a78c3` and `3fd8a91` apply clean outright (the + plan's own earlier hand-edits align their contexts). `46d3a61` — assumed clean — + conflicts in `src/config.py` against copi-prod's `audit_recipient_list` property; + the `_guard_rate_limiter_settings` validator arrives in `46d3a61`, not `9932645`. +- **§4 Phase-4 trim.** "There is nothing to lint" was wrong: `from pathlib import + Path` is present-and-unused from `46a8391` onward, and `3a23e73`'s removal of it + must be taken or `ci.sh`'s tests-at-zero gate fails at every checkpoint. +- **Hand-applied D** covers **four** call sites, not three: the phase-5 new-post + guard is `29fc8f1` caller-side code, which the `-> bool` port deliberately leaves + behind. +- **Phase 0.** The post-merge added-file check shows exactly the two onboarding + templates — no pre-0019 alembic files appear. The merge also (a) carries + copi-prod's `prompts/daily_audit.md` edit (`9ab5555`), so every "prompts/ diff is + identity.md only" check reads two lines, and (b) collides *semantically* with + `tests/unit/test_config_secret_redaction.py`: the new `audit_recipients` field + must be classified non-secret or two tests fail with no textual conflict to warn + anyone. +- **§6.** The `f2cbfe9` claim was half right: the token-absence assertion passes, + but its four menu-presence siblings cannot; they are dropped with a `Ported-from: + f2cbfe9 (partial)` trailer. `test_admin_can_set_agent_role` (from `4ec8ab7`) + needs a tmp roles dir for the same structural reason — `available_roles()` has no + scout_hub here, and the route correctly refuses the write. +- **Counts.** Expect ~74 commits in `rev-list cohort..HEAD` (5 docs + 65 port-side + + copi-prod's 4, which the merge brings into the count) and **10** `Ported-from:` + trailers citing eleven blackbird shas. diff --git a/prompts/identity.md b/prompts/identity.md new file mode 100644 index 0000000..27a4b00 --- /dev/null +++ b/prompts/identity.md @@ -0,0 +1,3 @@ +## Your Identity +You are **{bot_name}**, the AI agent representing the {pi_name} lab at Scripps Research. +Your agent ID is "{agent_id}". When communicating, represent your lab professionally. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c354b82..4555ccb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,15 @@ dev = [ # mutmut 3.x's instrumentation trampoline rejects this repo's `src.`-prefixed # module names; 2.x mutates files directly and works with the src/ layout. "mutmut>=2.4,<3", + # For the human/driven half of tests/e2e only — the browser that replays + # tests/e2e/test_browser_flows.py::FLOWS and drops its screenshots in + # .playwright-mcp/. NOTHING under tests/ imports it: the pytest tier in that + # directory is httpx replays of the same route sequences, so `pytest tests/` + # is green without it and the wheel alone is not enough to drive a browser — + # `python -m playwright install chromium` fetches the ~300MB binaries, which + # pip cannot. Pinned as a floor, not an upper bound; the browser build is + # chosen by that command, not by this line. + "playwright>=1.44.0", ] [project.scripts] @@ -65,15 +74,36 @@ markers = [ "integration: needs a real Postgres (testcontainers) + Docker", "characterization: golden-master snapshot test", "contract: respx-mocked external HTTP", + "real_llm: spends real Anthropic tokens; skipped unless ANTHROPIC_API_KEY is set", + "live_slack: hits a real Slack workspace; needs SLACK_TEST_WORKSPACE=1 plus bot tokens in the environment", + "live_api: calls a real third-party API (ORCID/NCBI/grants.gov); needs LIVE_API_TESTS=1", ] [tool.coverage.run] branch = true source = ["src"] +# Without this, every coverage number this project has ever reported is wrong — +# understated, and unevenly so. SQLAlchemy's asyncio layer runs ORM work inside +# greenlets, and coverage's tracer loses the frame across a greenlet switch. The +# symptom is precise: in an async route handler, the statement holding the FIRST +# `await db.execute(...)` is recorded and everything after it is reported missing, +# even in handlers the tests demonstrably drive to completion. Measured on +# src/routers/admin.py against the same suite and the same commit: 19% without +# this line, 41% with it (484 vs 359 statements missed). So the shortfall is +# largest in exactly the modules that talk to the database most. +# +# Re-measured 2026-08-04 with this line in place: the suite's true total is 61.638%, +# not the 35.66% the suppressed data reported. COV_MIN in scripts/ci.sh has been +# re-baselined from 35 to 60 accordingly. +concurrency = ["thread", "greenlet"] [tool.coverage.report] show_missing = false -precision = 0 +# Two decimals, not 0. At precision = 0 the total printed as "62%" while the real +# figure was 61.638% — i.e. the report rounded UP past the true value, so anyone +# re-baselining COV_MIN off the printed number would set a floor the suite cannot +# actually meet. A ratchet is only as good as the number you read it from. +precision = 2 # Mutation-testing scope lives in scripts/mutation.sh (CLI flags), not here — mutmut # 2.x reads setup.cfg/CLI rather than pyproject's [tool.mutmut]. diff --git a/scripts/audit_pub_dois.py b/scripts/audit_pub_dois.py index 52e72d3..acf09ff 100644 --- a/scripts/audit_pub_dois.py +++ b/scripts/audit_pub_dois.py @@ -19,15 +19,15 @@ Usage (runs inside the app container — needs DB + network): # Audit everyone (report only): - docker exec copi-python-app-1 python scripts/audit_pub_dois.py + docker compose exec app python scripts/audit_pub_dois.py # Audit + fix specific users by ORCID: - docker exec copi-python-app-1 python scripts/audit_pub_dois.py \\ + docker compose exec app python scripts/audit_pub_dois.py \ --orcids 0000-0000-0000-0001 --fix # Audit + fix specific agents, or everyone: - docker exec copi-python-app-1 python scripts/audit_pub_dois.py --agents liu bollong --fix - docker exec copi-python-app-1 python scripts/audit_pub_dois.py --fix + docker compose exec app python scripts/audit_pub_dois.py --agents liu bollong --fix + docker compose exec app python scripts/audit_pub_dois.py --fix """ from __future__ import annotations diff --git a/scripts/backfill_agent_tokens.py b/scripts/backfill_agent_tokens.py index a0c6cf3..996e9f9 100644 --- a/scripts/backfill_agent_tokens.py +++ b/scripts/backfill_agent_tokens.py @@ -11,9 +11,9 @@ Usage (inside the app container): - docker exec copi-python-app-1 python scripts/backfill_agent_tokens.py + docker compose exec app python scripts/backfill_agent_tokens.py # preview only: - docker exec copi-python-app-1 python scripts/backfill_agent_tokens.py --dry-run + docker compose exec app python scripts/backfill_agent_tokens.py --dry-run """ from __future__ import annotations diff --git a/scripts/backfill_agents.py b/scripts/backfill_agents.py index e9eee23..34be470 100644 --- a/scripts/backfill_agents.py +++ b/scripts/backfill_agents.py @@ -8,8 +8,8 @@ Usage (inside the app container): - docker cp scripts/backfill_agents.py copi-python-app-1:/app/scripts/ - docker exec copi-python-app-1 python scripts/backfill_agents.py \\ + docker compose cp scripts/backfill_agents.py app:/app/scripts/ + docker compose exec app python scripts/backfill_agents.py \ --orcids data/cohorts/newuserlist01_orcids.txt The --orcids file is the same format as orcids.txt: one ORCID per line, diff --git a/scripts/backfill_slack_history_to_db.py b/scripts/backfill_slack_history_to_db.py new file mode 100644 index 0000000..07212a0 --- /dev/null +++ b/scripts/backfill_slack_history_to_db.py @@ -0,0 +1,125 @@ +"""One-time backfill: import current Slack conversation history into the DB. + +Since the DB became the primary conversation store (specs/local-db-conversations.md), +agent_messages carries message content. Historically content lived only in Slack, +so pre-cutover runs have metadata-only rows. Run this once, with Slack tokens +available, to pull the workspace's channel + thread history into agent_messages +(content + slack_ts as the canonical message_ts) before switching to DB-primary +operation, preserving in-flight conversations. + +It reuses the engine's own setup/rebuild machinery (seeded + private channels, +the Slack reconcile, and the persist flush), then exits — it does NOT run any +agent turns or make LLM calls. + +Idempotent: the reconcile appends only messages not already in the DB, and the +flush upserts on (simulation_run_id, message_ts). Safe to re-run. + +Usage (inside the app container): + + docker compose exec app python scripts/backfill_slack_history_to_db.py +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from sqlalchemy import desc, func, select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.config import get_settings +from src.models import AgentMessage, AgentRegistry, SimulationRun +from src.services.slack_tokens import env_token, is_valid_token + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger("backfill_slack_history") + + +async def main(run_id_arg: str | None) -> None: + settings = get_settings() + engine = create_async_engine(settings.database_url) + sf = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + # Roster (active agents) + tokens, mirroring src/agent/main.py. + async with sf() as db: + rows = (await db.execute( + select( + AgentRegistry.agent_id, AgentRegistry.bot_name, + AgentRegistry.pi_name, AgentRegistry.slack_bot_token, + ).where(AgentRegistry.status == "active").order_by(AgentRegistry.agent_id) + )).all() + if run_id_arg: + run_id = run_id_arg + else: + run_id = (await db.execute( + select(SimulationRun.id).order_by(desc(SimulationRun.started_at)).limit(1) + )).scalar_one_or_none() + + if run_id is None: + logger.error("No SimulationRun found — start a run first (nothing to attach to).") + await engine.dispose() + return + + agents = [Agent(agent_id=r.agent_id, bot_name=r.bot_name, pi_name=r.pi_name) for r in rows] + + from src.agent.slack_client import AgentSlackClient + slack_clients = {} + for r in rows: + tok = r.slack_bot_token if is_valid_token(r.slack_bot_token) else env_token(r.agent_id) + if is_valid_token(tok): + client = AgentSlackClient(agent_id=r.agent_id, bot_token=tok) + if client.connect(): + slack_clients[r.agent_id] = client + if not slack_clients: + logger.error("No connected Slack clients — cannot backfill from Slack.") + await engine.dispose() + return + + async with sf() as db: + before = (await db.execute( + select(func.count(AgentMessage.id)).where( + AgentMessage.simulation_run_id == run_id, + func.length(AgentMessage.content) > 0, + ) + )).scalar_one() + + sim = SimulationEngine( + agents=agents, slack_clients=slack_clients, session_factory=sf, + simulation_run_id=run_id, slack_enabled=True, + ) + # Reuse the engine's setup + rebuild, then flush to the DB. No turns run. + sim._ensure_seeded_channels() + await sim._persist_seeded_channels() + await sim._sync_private_channels_from_db() + sim.message_log.set_persist_callback(sim._enqueue_persist) + await sim._rebuild_state_from_db() + await sim._rebuild_state_from_slack() + await sim._flush_persisted() + + async with sf() as db: + after = (await db.execute( + select(func.count(AgentMessage.id)).where( + AgentMessage.simulation_run_id == run_id, + func.length(AgentMessage.content) > 0, + ) + )).scalar_one() + + logger.info( + "Backfill complete for run %s: content-bearing messages %d -> %d (log holds %d).", + run_id, before, after, len(sim.message_log), + ) + await engine.dispose() + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--run-id", default=None, help="Target SimulationRun id (default: latest)") + args = ap.parse_args() + asyncio.run(main(args.run_id)) diff --git a/scripts/backfill_slack_ts.py b/scripts/backfill_slack_ts.py new file mode 100644 index 0000000..f793ef2 --- /dev/null +++ b/scripts/backfill_slack_ts.py @@ -0,0 +1,157 @@ +"""One-time repair of the ``agent_messages.slack_ts`` mirror mapping. + +Rows written before Stage 6 (and, until the poller's bot branch was fixed, any +message polled from another workspace bot) came from Slack but were stored with +``slack_ts`` NULL. ``_restored_slack_ts`` used to paper over that by *inferring* +the mapping — "a row in a real Slack channel was born on Slack, so its canonical +id is its Slack ts" — which is wrong for a DB-origin message that also carries a +real Slack channel id: a PI message written through the web inbox, or an agent +post whose Slack mirror failed. Inferring there fabricates a timestamp Slack +never issued, and the engine then hands it to chat.postMessage as a thread_ts. + +So the guess is gone and this script repairs the data instead, by asking Slack +which timestamps actually exist. Only confirmed ones are written; anything Slack +does not recognise is left NULL, which is now the truthful value. + +Run it once per deployment that has pre-Stage-6 history, BEFORE relying on the +no-inference behaviour: + + docker compose exec app python scripts/backfill_slack_ts.py # report only + docker compose exec app python scripts/backfill_slack_ts.py --apply # write + +Read-only against Slack; the only DB writes are ``slack_ts`` on rows Slack +confirmed. Safe to re-run. +""" + +from __future__ import annotations + +import asyncio +import sys + +from slack_sdk import WebClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from src.config import get_settings +from src.services.slack_tokens import get_any_bot_token + +CANDIDATES = text( + """ + SELECT message_ts, channel_id, channel_name, sender_name, thread_ts + FROM agent_messages + WHERE slack_ts IS NULL + AND channel_id NOT LIKE 'local:%' + -- A NULL message_ts has nothing to look up. Included previously, which sent + -- conversations.history(latest=None, oldest=None) — that returns the channel's + -- NEWEST message, compares it to None, and books the row as DB-origin. One + -- wasted Slack call per row and an inflated db_origin count. + AND message_ts IS NOT NULL + ORDER BY message_ts + """ +) + +APPLY = text( + """ + UPDATE agent_messages SET slack_ts = message_ts + WHERE slack_ts IS NULL AND message_ts = :ts AND channel_id = :ch + """ +) + + +def _exists_on_slack( + client: WebClient, channel_id: str, ts: str, thread_ts: str | None = None +) -> bool | None: + """True/False if Slack answered, None if the lookup itself failed. + + A thread reply MUST be looked up with conversations.replies. + ``conversations.history`` does not return replies at all, so asking it about one + yields an empty page — indistinguishable from "this timestamp does not exist". + That made the old single-call version report every thread reply as + ``NOT on Slack (DB-origin)`` and leave its slack_ts NULL forever: not an + "unverified" it could be retried from, but a confident wrong answer. Measured + against a real reply in this workspace: history returned [], replies returned it. + Agents converse almost entirely in threads, so that was most of the rows. + """ + try: + if thread_ts: + resp = client.conversations_replies( + channel=channel_id, ts=thread_ts, latest=ts, oldest=ts, + inclusive=True, limit=100, + ) + else: + resp = client.conversations_history( + channel=channel_id, latest=ts, oldest=ts, inclusive=True, limit=1, + ) + except Exception as exc: # noqa: BLE001 — an API error must not be read as "absent" + print(f" ! lookup failed for {ts} in {channel_id}: {exc}") + return None + return any(m.get("ts") == ts for m in resp.get("messages", [])) + + +async def main(apply: bool) -> int: + settings = get_settings() + engine = create_async_engine(settings.database_url) + session_factory = async_sessionmaker(engine, expire_on_commit=False) + + async with session_factory() as db: + rows = (await db.execute(CANDIDATES)).all() + token = await get_any_bot_token(db) + + if not rows: + print("Nothing to do: no rows with a NULL slack_ts in a Slack channel.") + await engine.dispose() + return 0 + if not token: + print("ERROR: no usable bot token — cannot verify against Slack.", file=sys.stderr) + await engine.dispose() + return 1 + + client = WebClient(token=token) + confirmed: list[tuple[str, str]] = [] + absent = 0 + errored = 0 + print(f"{len(rows)} candidate row(s):\n") + for ts, channel_id, channel_name, sender_name, thread_ts in rows: + found = _exists_on_slack(client, channel_id, ts, thread_ts) + mark = {True: "on Slack", False: "NOT on Slack (DB-origin)", None: "unverified"}[found] + print(f" {ts} #{channel_name:<24} {sender_name:<22} {mark}") + if found is True: + confirmed.append((ts, channel_id)) + elif found is False: + absent += 1 + else: + errored += 1 + + print( + f"\nconfirmed={len(confirmed)} db_origin={absent} unverified={errored}" + ) + if not apply: + print("\nDry run. Re-run with --apply to write slack_ts on the confirmed rows.") + if errored: + print( + f"WARNING: {errored} row(s) UNVERIFIED — fix access before --apply, or " + "they stay NULL.", file=sys.stderr, + ) + await engine.dispose() + return 2 if errored else 0 + + async with session_factory() as db: + for ts, channel_id in confirmed: + await db.execute(APPLY, {"ts": ts, "ch": channel_id}) + await db.commit() + # Do NOT claim the remainder is "correct": an unverified row is a row we could + # not ask about, not a row Slack denied. The previous wording printed + # "which is correct" on runs where every single lookup had failed. + print(f"\nUpdated {len(confirmed)} row(s). {absent} denied by Slack (correctly NULL).") + if errored: + print( + f"WARNING: {errored} row(s) UNVERIFIED — Slack could not be asked (token " + "scope, bot not in channel, or rate limit). Re-run; this is not a verdict.", + file=sys.stderr, + ) + await engine.dispose() + return 2 if errored else 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main(apply="--apply" in sys.argv))) diff --git a/scripts/build_cabo_sankey.py b/scripts/build_cabo_sankey.py index f7ba0ff..520d569 100644 --- a/scripts/build_cabo_sankey.py +++ b/scripts/build_cabo_sankey.py @@ -1,22 +1,22 @@ -"""Sankey funnel: top-level posts → threads → outcomes, for a simulation cohort. +"""Sankey funnel: top-level posts → threads → outcomes, for one simulation run window. Pulls numbers live from Postgres so it can be re-run as the simulation -progresses. Parameterized by cohort start date so it serves any cohort that +progresses. Parameterized by window start date so it serves any run window that shares the single resumed simulation_run_id (date is the only way to isolate a -cohort — see src/routers/public.py and memory project_reunion_cohort_boundary). +window — see the window constants in src/routers/public.py). Run inside the app container (scripts/ isn't mounted — docker cp it in first): - docker cp scripts/build_cabo_sankey.py copi-python-app-1:/app/scripts/ + docker compose cp scripts/build_cabo_sankey.py app:/app/scripts/ # Cabo run (defaults): - docker exec copi-python-app-1 python scripts/build_cabo_sankey.py + docker compose exec app python scripts/build_cabo_sankey.py - # Schultz alumni reunion cohort: - docker exec copi-python-app-1 python scripts/build_cabo_sankey.py \ + # Schultz alumni reunion window: + docker compose exec app python scripts/build_cabo_sankey.py \ --start 2026-06-06 --out /app/data/schultz_viz --label "Schultz Alumni reunion run" Output (sankey.html + sankey.png) lands in --out inside the container; retrieve -with `docker cp copi-python-app-1:/app/data/schultz_viz ./data/`. +with `docker cp app:/app/data/schultz_viz ./data/`. """ from __future__ import annotations @@ -143,7 +143,7 @@ async def main() -> None: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--start", default=DEFAULT_START, - help=f"Cohort start date (ISO, UTC). Default {DEFAULT_START}.") + help=f"Run-window start date (ISO, UTC). Default {DEFAULT_START}.") ap.add_argument("--out", default=DEFAULT_OUT, help=f"Output dir (inside container). Default {DEFAULT_OUT}.") ap.add_argument("--label", default=DEFAULT_LABEL, diff --git a/scripts/ci.sh b/scripts/ci.sh index 643cd4b..e514319 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -5,28 +5,78 @@ # GitHub-side hooks by design — this script is the whole gate, and it runs on push. # # Steps: -# 1. ruff lint of the test suite. (New test code is kept clean. Legacy src/ carries -# pre-existing style debt — out of scope for this behavior-pinning gate; lint it -# separately with `ruff check src` when you're ready to pay that down.) -# 2. Full pytest run — unit + integration + characterization + contract — with +# 1. Alembic sanity: exactly one head, no duplicate revision ids. Cheap, offline, +# and first because it catches the one class of breakage that a clean `git merge` +# and a fully green test suite both miss. See .notes/cohort-system-v2.md §14. +# 2. Alembic round trip: upgrade -> downgrade -> upgrade against a THROWAWAY +# Postgres that this step starts and destroys itself. On by default since +# 2026-08-04; set CI_MIGRATION_DB=none to skip. +# 3. ruff lint of the test suite. New test code is kept spotless — zero findings. +# 4. ruff lint of src/ against a CEILING (SRC_LINT_MAX) rather than zero. src/ +# carries pre-existing style debt, so this is a ratchet: it blocks NEW debt +# without demanding the old debt be paid first. +# 5. Full pytest run — unit + integration + characterization + contract — with # branch coverage over src/, failing under COV_MIN (a ratchet floor: raise it as # coverage grows, never lower it). # # The integration/characterization/contract suites spin an ephemeral Postgres via # testcontainers, so a reachable Docker daemon is required. # -# Overridable env: VENV_PY (python interpreter), COV_MIN (coverage floor %). +# Overridable env: VENV_PY (python interpreter), COV_MIN (coverage floor %), +# SRC_LINT_MAX (src/ lint ceiling), CI_MIGRATION_DB (round-trip DSN, or `none` to +# skip the round trip), MIGCHECK_PORT (host port for the throwaway Postgres), +# MIGRATION_FLOOR (the revision the round trip downgrades to). set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$REPO_ROOT" VENV_PY="${VENV_PY:-$REPO_ROOT/.venv-test/bin/python}" -COV_MIN="${COV_MIN:-35}" # measured 35.66% on this suite; raise as coverage grows, never lower +# Coverage floor. Re-baselined 35 -> 60 on 2026-08-04. The old 35 was not a judgement +# about this suite; it was measured at 35.66% against a broken tracer. bd68fae added +# `concurrency = ["thread", "greenlet"]` to [tool.coverage.run] — without it coverage +# loses the frame across SQLAlchemy's greenlet switch and stops recording an async +# handler at its first `await db.execute(...)`. The true figure on the same suite and +# the same commit is 61.638% (1167 passed, 120 skipped). 60 leaves ~1.6 points of slack +# so the gate does not go red on ordering noise. Raise as coverage grows, never lower. +COV_MIN="${COV_MIN:-60}" + +# Ceiling on ruff findings in src/, NOT a target. Measured 2026-08-04 with the same +# command the ratchet below runs, so the numbers are comparable: origin/main 292, this +# branch's pre-repair tip (8515f65) 308, HEAD 260. +# +# LOWER THIS AS DEBT IS PAID; NEVER RAISE IT. Raising it to make a push go through is +# precisely how those 16 findings got into admin.py in the first place — a ceiling that +# moves up to meet the code is not a gate, it is a logbook. +SRC_LINT_MAX="${SRC_LINT_MAX:-260}" + +# Throwaway-Postgres settings for the migration round trip (step 2). The port is +# published on 127.0.0.1 only. MIGRATION_FLOOR is how far down the round trip goes; +# lowering it widens the round trip, which is always safe on a throwaway database. +MIGCHECK_PORT="${MIGCHECK_PORT:-55432}" +MIGCHECK_CONTAINER="copi-ci-migcheck" +# 0018, not 0021. At 0021 the round trip never executed the 0019/0020/0021 +# DOWNGRADES — and those are the ones with teeth: 0019's downgrade drops the +# content columns and puts agent_id back to NOT NULL, and 0019/0020/0021 lack the +# if_exists guards that 0022/0023 have. The gate runs against an empty throwaway +# database, so the NOT NULL step cannot fail here; that is precisely why it is safe +# to exercise, and why the gate still cannot catch the data-dependent failure that +# same step produces on a populated production database. Raise this only to skip +# work deliberately. +MIGRATION_FLOOR="${MIGRATION_FLOOR:-0018}" LINT_TARGETS=( tests/conftest.py tests/factories.py tests/fakes.py tests/unit tests/integration tests/characterization tests/contract + # tests/e2e was the one test directory the gate never linted. Added 2026-08-04, + # when that tier was first run end to end; it was already at zero findings, so + # this closes the hole without paying anything down. Two of its nine tests need + # no server and run in the offline suite, so it is gate-relevant either way. + tests/e2e + # The production migration tooling. Not tests, but it is the code an operator runs + # against a live database during an outage window, so it gets held to the same bar. + # Verified at zero findings when added 2026-08-04. + scripts/migrate ) if [ ! -x "$VENV_PY" ]; then @@ -42,9 +92,163 @@ if ! docker info >/dev/null 2>&1; then exit 1 fi +echo "==> alembic (single head, no duplicate revision ids)" +# Two migrations sharing a revision id is invisible to git and to pytest: the merge +# is clean, every test passes, and Alembic only warns. The damage shows up at deploy +# — `alembic upgrade head` dies on multiple heads, and a targeted `upgrade ` +# silently applies whichever duplicate sorts last while stamping the DB as fully +# migrated. Assign revision ids at merge, never at branch. +dupes="$(grep -h '^revision' alembic/versions/*.py | sort | uniq -d || true)" +if [ -n "$dupes" ]; then + echo "ERROR: duplicate alembic revision ids:" >&2 + echo "$dupes" >&2 + grep -l "^revision" alembic/versions/*.py | while read -r f; do + printf ' %s -> %s\n' "$f" "$(grep -m1 '^revision' "$f")" >&2 + done + exit 1 +fi +# `alembic heads` reads only the script directory — no database needed. +heads_out="$("$VENV_PY" -m alembic heads 2>/dev/null || true)" +heads_n="$(printf '%s\n' "$heads_out" | grep -c '[^[:space:]]' || true)" +if [ "$heads_n" -ne 1 ]; then + echo "ERROR: expected exactly 1 alembic head, found ${heads_n}:" >&2 + printf '%s\n' "$heads_out" >&2 + echo "Renumber the newer migration onto the current head before merging." >&2 + exit 1 +fi +echo " single head: $(printf '%s\n' "$heads_out" | tr -d '\n')" + +# Round trip against a THROWAWAY database. ON BY DEFAULT since 2026-08-04. +# +# The unit tests already pin the migrations' static properties (single head, no +# duplicate ids, every drop guarded with if_exists). What this adds is the one thing +# static analysis cannot show: that upgrade -> downgrade -> upgrade actually RUNS +# clean. It was gated behind an unset CI_MIGRATION_DB for the entire life of the +# cohort branch, so 0022 and 0023 were never round-tripped by the gate at all. +# +# The step brings its own database SERVER — a throwaway postgres:15 container whose +# port is published on 127.0.0.1 and which is destroyed by the EXIT trap below. That +# is not gold-plating; it is the only DSN that actually works here. `ci.sh` runs on +# the HOST, and both obvious DSNs are wrong: +# +# * `...@postgres:5432/...` is the compose-INTERNAL hostname. On the host it either +# does not resolve, or — verified 2026-08-04 on this developer's machine — it +# resolves to an unrelated real server (`postgres.int.hueb.org`) via the LAN's +# search domain. A migration round trip that DROPs and re-CREATEs schema must +# never be one DNS record away from someone else's database. +# * `...@localhost:5432/...` is refused: docker-compose.yml publishes NO host port +# for the postgres service (only app's 8001), so the dev database is simply not +# reachable from the host. This is what the plan for this change assumed, and it +# does not work. +# +# Publishing our own port makes `localhost` true by construction, and owning the +# server means this step cannot touch the dev database even in principle. +# +# CI_MIGRATION_DB=none skips the step. CI_MIGRATION_DB= runs it against a +# database you supply instead of the throwaway container — NEVER point that at a +# database with data you want. +migcheck_cleanup() { docker rm -f "$MIGCHECK_CONTAINER" >/dev/null 2>&1 || true; } + +if [ "${CI_MIGRATION_DB:-}" = "none" ]; then + echo "==> alembic round trip SKIPPED (CI_MIGRATION_DB=none)" +else + if [ -n "${CI_MIGRATION_DB:-}" ]; then + migration_dsn="$CI_MIGRATION_DB" + echo "==> alembic round trip against caller-supplied $migration_dsn" + else + migration_dsn="postgresql+asyncpg://copi:copi@127.0.0.1:${MIGCHECK_PORT}/copi_migcheck" + echo "==> alembic round trip against a throwaway postgres:15 on 127.0.0.1:${MIGCHECK_PORT}" + # Fixed container name, removed up front as well as on exit, so a run that was + # killed mid-flight cannot wedge the next one. ci.sh is a serial pre-push gate; + # two concurrent runs would collide on the port regardless of the name. + # INT and TERM as well as EXIT: this gate runs for ~6 minutes, so Ctrl-C + # partway through is the likely case, and a leaked container keeps + # MIGCHECK_PORT bound — the next run would then fail its readiness wait and + # look like a broken migration rather than a stale container. + trap migcheck_cleanup EXIT INT TERM + migcheck_cleanup + docker run -d --name "$MIGCHECK_CONTAINER" \ + -e POSTGRES_USER=copi -e POSTGRES_PASSWORD=copi -e POSTGRES_DB=copi_migcheck \ + -p "127.0.0.1:${MIGCHECK_PORT}:5432" postgres:15 >/dev/null + migcheck_ready=0 + for _ in $(seq 1 60); do + if docker exec "$MIGCHECK_CONTAINER" pg_isready -U copi -q >/dev/null 2>&1; then + migcheck_ready=1 + break + fi + sleep 1 + done + if [ "$migcheck_ready" -ne 1 ]; then + echo "ERROR: throwaway postgres on 127.0.0.1:${MIGCHECK_PORT} never became ready." >&2 + echo "Is that port already in use? Override with MIGCHECK_PORT=." >&2 + docker logs "$MIGCHECK_CONTAINER" 2>&1 | tail -20 >&2 + exit 1 + fi + echo " throwaway postgres ready" + fi + DATABASE_URL="$migration_dsn" "$VENV_PY" -m alembic upgrade head + DATABASE_URL="$migration_dsn" "$VENV_PY" -m alembic downgrade "$MIGRATION_FLOOR" + DATABASE_URL="$migration_dsn" "$VENV_PY" -m alembic upgrade head + echo " round trip clean (upgrade head -> downgrade ${MIGRATION_FLOOR} -> upgrade head)" + # Tear down now rather than at exit: pytest below spins its own Postgres via + # testcontainers and takes minutes, and there is no reason to hold a second server + # and a bound port for all of it. The EXIT trap stays armed as the failure path. + if [ -z "${CI_MIGRATION_DB:-}" ]; then + migcheck_cleanup + echo " throwaway postgres destroyed" + fi +fi + echo "==> ruff (test-suite lint)" "$VENV_PY" -m ruff check "${LINT_TARGETS[@]}" +echo "==> ruff (src/ ratchet, ceiling ${SRC_LINT_MAX})" +# A ceiling, not zero: src/ carries pre-existing style debt, and demanding it all be +# paid before the next push would just get this gate deleted. What the ceiling buys is +# that NEW debt cannot get in — the cohort branch put 16 findings into admin.py past a +# gate that only ever linted tests/. +# +# Three details that are easy to get wrong here, all of them load-bearing: +# * --quiet suppresses ruff's trailing "Found N errors." / "[*] N fixable" summary. +# Without it those two lines are counted as findings and every number is +2. +# * ruff exits 1 when it finds anything, and this script runs under `set -o pipefail`, +# so a bare `ruff ... | wc -l` command substitution aborts the whole script at the +# assignment, silently and with no message. Hence the explicit rc capture. +# * exit >1 means ruff itself failed (a malformed config, for instance — verified to +# exit 2). Treat that as a gate failure, never as "zero findings"; a ratchet that +# fails open is worse than no ratchet. +set +e +src_lint_out="$("$VENV_PY" -m ruff check src --output-format=concise --quiet 2>&1)" +src_lint_rc=$? +set -e +if [ "$src_lint_rc" -gt 1 ]; then + echo "ERROR: ruff failed to run over src/ (exit ${src_lint_rc}):" >&2 + printf '%s\n' "$src_lint_out" >&2 + exit 1 +fi +# The rc check above is not enough on its own. A missing or unreadable path is NOT an +# error exit: ruff emits a single E902 diagnostic and still exits 1, so an I/O problem +# is indistinguishable from "that file has one finding" — and it makes the count go +# DOWN. Measured 2026-08-04: `chmod 000 src/routers/admin.py` takes the total from 260 +# to 193, because that file's 68 findings disappear and one E902 replaces them. The +# ratchet would pass, and the next person would "helpfully" re-baseline the ceiling to +# 193 and lock the loss in. So refuse to produce a number at all. +if printf '%s' "$src_lint_out" | grep -q 'E902'; then + echo "ERROR: ruff could not read part of src/ (E902), so the finding count is not a" >&2 + echo "measurement. Fix the path or the permissions. Do NOT re-baseline SRC_LINT_MAX" >&2 + echo "off a run that reported this." >&2 + printf '%s\n' "$src_lint_out" | grep 'E902' >&2 + exit 1 +fi +src_findings="$(printf '%s' "$src_lint_out" | grep -c . || true)" +if [ "$src_findings" -gt "$SRC_LINT_MAX" ]; then + echo "ERROR: ruff findings in src/ rose to ${src_findings}; the ceiling is ${SRC_LINT_MAX}." >&2 + echo "Fix what you added. Do not raise SRC_LINT_MAX in scripts/ci.sh to make this pass." >&2 + printf '%s\n' "$src_lint_out" >&2 + exit 1 +fi +echo " ${src_findings} findings (ceiling ${SRC_LINT_MAX})" + echo "==> pytest (full suite + branch coverage, fail-under=${COV_MIN}%)" "$VENV_PY" -m pytest tests/ \ --cov=src --cov-report=term-missing \ diff --git a/scripts/export_agent_roster.py b/scripts/export_agent_roster.py index fa3d493..4ddfe26 100644 --- a/scripts/export_agent_roster.py +++ b/scripts/export_agent_roster.py @@ -15,7 +15,7 @@ python scripts/export_agent_roster.py # or simply: - docker exec copi-python-app-1 python scripts/export_agent_roster.py + docker compose exec app python scripts/export_agent_roster.py """ from __future__ import annotations diff --git a/scripts/generate_sparsedata_user.py b/scripts/generate_sparsedata_user.py index bcf87f3..d03562f 100644 --- a/scripts/generate_sparsedata_user.py +++ b/scripts/generate_sparsedata_user.py @@ -19,8 +19,8 @@ Usage (runs inside the app container — needs DB + prompts + profiles): - docker cp scripts/generate_sparsedata_user.py copi-python-app-1:/app/scripts/ - docker exec copi-python-app-1 python scripts/generate_sparsedata_user.py \\ + docker compose cp scripts/generate_sparsedata_user.py app:/app/scripts/ + docker compose exec app python scripts/generate_sparsedata_user.py \ --file data/cohorts/newuserlist02.tsv --force Outputs: diff --git a/scripts/migrate/postflight.py b/scripts/migrate/postflight.py new file mode 100644 index 0000000..948137b --- /dev/null +++ b/scripts/migrate/postflight.py @@ -0,0 +1,786 @@ +#!/usr/bin/env python3 +"""Post-migration verification for the 0018/0019 -> 0023 upgrade. + +Run this AFTER `alembic upgrade`, against the database you just migrated. + + docker compose exec -T -e DATABASE_URL=... app python scripts/migrate/postflight.py \ + --snapshot /app/logs/migration_snapshot.json + +Exit codes (contract): + + 0 verified + 1 verification FAILED + +Why this script exists at all, given that `alembic upgrade` exits 0 on success: +because `alembic upgrade` exiting 0 does not mean the schema changed. + + * A duplicate revision id makes a targeted ``upgrade `` apply whichever file + sorts last while stamping the database as fully migrated. This repo has had two + files claiming ``revision = "0019"`` (``0019_agent_message_content.py`` and, on + branch cohort-agent-isolation, ``0019_add_cohorts.py``). + * If ``alembic/env.py`` emits SQL on the connection before + ``context.begin_transaction()``, alembic hands the commit back to the caller, + which never commits: the log shows the whole chain "Running upgrade ..." and the + database is left untouched. Reproduced on this tree. + +So the version stamp is evidence of nothing on its own. Every check below looks at +the actual catalog, the actual data, or the actual ORM. + +Warnings are reported but do not fail the run (exit stays 0); only a FAIL exits 1. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + # See the identical note in preflight.py: without this, `import src...` resolves to + # the stale copy baked into site-packages by the Dockerfile's `pip install .`. + sys.path.insert(0, str(REPO_ROOT)) + + +def _load_preflight(): + """Import preflight.py by path. + + scripts/ is not a package (no __init__.py anywhere in it), so a plain + `from preflight import ...` only works when CWD happens to be scripts/migrate. + """ + name = "copi_migrate_preflight" + already = sys.modules.get(name) + if already is not None: + # Idempotent: return the module that is already registered rather than exec'ing a + # second copy. Two live copies would leave preflight's @dataclass types resolving + # their annotations against the OTHER copy's globals. + return already + path = Path(__file__).resolve().parent / "preflight.py" + spec = importlib.util.spec_from_file_location(name, path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + # Registered BEFORE exec_module, not after: @dataclass resolves annotations through + # sys.modules[cls.__module__], and preflight.py defines three dataclasses. + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +_pf = _load_preflight() + +PASS = _pf.PASS +WARN = _pf.WARN +FAIL = _pf.BLOCK # same token, read as "verification failed" here + +DEFAULT_TARGET = _pf.DEFAULT_TARGET +Report = _pf.Report +compare_row_counts = _pf.compare_row_counts +fetch_all = _pf.fetch_all +fetch_one_value = _pf.fetch_one_value +current_revision = _pf.current_revision +open_connection = _pf.open_connection +redact_url = _pf.redact_url +resolve_database_url = _pf.resolve_database_url +table_exists = _pf.table_exists +add_common_arguments = _pf.add_common_arguments +check_alembic_scripts = _pf.check_alembic_scripts + +# --------------------------------------------------------------------------- +# What "migrated to 0023" actually means, object by object. +# Captured from a database that reached 0023 through the real chain, then pinned here. +# --------------------------------------------------------------------------- + +#: (table, column, data_type, is_nullable, column_default) +#: column_default None means "any default is acceptable"; '' means "must have none". +EXPECTED_COLUMNS: tuple[tuple[str, str, str, bool, str | None], ...] = ( + # 0019 content columns. NOT NULL with a server_default is what makes 0019 fast on a + # big table (Postgres 11+ fills a non-volatile default without a rewrite) AND what + # makes every legacy row read as an empty message. + ("agent_messages", "content", "text", False, "''::text"), + ("agent_messages", "sender_name", "character varying", False, "''::character varying"), + ("agent_messages", "is_bot", "boolean", False, "true"), + ("agent_messages", "posted_at", "double precision", False, "'0'::double precision"), + ("agent_messages", "slack_ts", "character varying", True, ""), + ("agent_messages", "slack_channel_id", "character varying", True, ""), + ("agent_messages", "slack_thread_ts", "character varying", True, ""), + # 0019 RELAXES this one. If it is still NOT NULL, 0019 did not really run. + ("agent_messages", "agent_id", "character varying", True, ""), + # 0020 + ("pi_dm_messages", "id", "uuid", False, None), + ("pi_dm_messages", "simulation_run_id", "uuid", False, None), + ("pi_dm_messages", "agent_id", "character varying", False, None), + ("pi_dm_messages", "pi_user_id", "character varying", False, None), + ("pi_dm_messages", "direction", "USER-DEFINED", False, None), + ("pi_dm_messages", "content", "text", False, None), + ("pi_dm_messages", "sender_name", "character varying", False, "''::character varying"), + ("pi_dm_messages", "ts", "character varying", False, None), + ("pi_dm_messages", "slack_ts", "character varying", True, ""), + ("pi_dm_messages", "posted_at", "double precision", False, "'0'::double precision"), + ("pi_dm_messages", "created_at", "timestamp with time zone", False, "now()"), + # 0023 — deliberately nullable and deliberately NOT backfilled. NULL means + # "this row predates the columns"; a non-null value here would be invented provenance. + ("researcher_profiles", "synthesis_validated", "boolean", True, ""), + ("researcher_profiles", "evidence_pmid_count", "integer", True, ""), + ("researcher_profiles", "evidence_pub_count", "integer", True, ""), + # 0024. NOT NULL with a server_default, so every existing agent reads as pi_lab + # — which is exactly the pre-0024 behaviour. + ("agents", "role", "character varying", False, "'pi_lab'::character varying"), +) + +EXPECTED_TABLES = ("pi_dm_messages", "cohorts", "cohort_memberships", "cohort_audit_events") + +#: The only tables the 0019..0023 chain creates, so the only ones legitimately absent +#: from a preflight row-count snapshot. Derived from preflight.PLANNED_OBJECTS rather +#: than re-listed, so the two cannot drift. +CHAIN_CREATED_TABLES = frozenset( + o.name for o in _pf.PLANNED_OBJECTS if o.kind == "table" +) + +#: index name -> the exact pg_indexes.indexdef tail, so a same-named index on the WRONG +#: columns (or a partial index that lost its predicate) fails too. +EXPECTED_INDEXES: dict[str, str] = { + "uq_agent_messages_run_ts": "USING btree (simulation_run_id, message_ts)", + "ix_agent_messages_run_posted": "USING btree (simulation_run_id, posted_at)", + "ix_agent_messages_run_channel_posted": ( + "USING btree (simulation_run_id, channel_name, posted_at)" + ), + "ix_agent_messages_run_slack_ts": ( + "USING btree (simulation_run_id, slack_ts) WHERE (slack_ts IS NOT NULL)" + ), + "ix_agent_messages_run_created": "USING btree (simulation_run_id, created_at)", + "ix_pi_dm_run_agent_posted": "USING btree (simulation_run_id, agent_id, posted_at)", + "ix_pi_dm_run_direction_posted": "USING btree (simulation_run_id, direction, posted_at)", + "ix_pi_dm_run_direction_created": "USING btree (simulation_run_id, direction, created_at)", + "ix_cohort_memberships_cohort_id": "USING btree (cohort_id)", + "ix_cohort_memberships_agent_id": "USING btree (agent_id)", + "ix_cohort_audit_events_cohort_id": "USING btree (cohort_id)", + "ix_cohort_audit_events_created_at": "USING btree (created_at)", + "uq_cohort_membership_cohort_agent": "USING btree (cohort_id, agent_id)", +} + +#: constraint name -> (table, pg_get_constraintdef) +EXPECTED_CONSTRAINTS: dict[str, tuple[str, str]] = { + "uq_agent_messages_run_ts": ( + "agent_messages", + "UNIQUE (simulation_run_id, message_ts)", + ), + "uq_cohort_membership_cohort_agent": ( + "cohort_memberships", + "UNIQUE (cohort_id, agent_id)", + ), +} + +EXPECTED_ENUMS: dict[str, tuple[str, ...]] = { + "pi_dm_direction_enum": ("inbound", "outbound"), +} + +#: Columns whose NULLs would be a data defect even though the catalog forbids them. +#: Checked in the data as well as in the catalog: a hand-relaxed column is exactly the +#: kind of drift this script is for. +MUST_BE_NON_NULL = tuple( + (t, c) for (t, c, _dt, nullable, _d) in EXPECTED_COLUMNS if not nullable +) + +# --------------------------------------------------------------------------- +# ORM-drift classification. +# --------------------------------------------------------------------------- +# alembic's compare_metadata() is a strong drift detector in ONE direction only. On a +# database that reached 0023 through the real chain it still reports 25 differences, +# every one of them the DB having something the ORM does not declare (indexes created in +# 0001-0017 with no Index() in the model, UniqueConstraints declared inline as +# unique=True, plus spurious add_table_comment entries). Those are pre-existing and +# harmless. The ops that mean "the DB is MISSING something the ORM requires" are the +# ones that matter, and they are the ones a dropped column/index/table produces: +# verified by sabotage — dropping agent_messages.content yields add_column, dropping +# ix_agent_messages_run_created yields add_index, and relaxing sender_name's NOT NULL +# yields modify_nullable. +DRIFT_FAIL_OPS = frozenset( + { + "add_table", + "add_column", + "add_index", + "add_constraint", + "modify_nullable", + "modify_type", + "remove_table", + } +) +DRIFT_IGNORED_OPS = frozenset( + {"remove_index", "remove_constraint", "add_table_comment", "remove_column"} +) + +# --------------------------------------------------------------------------- +# SQL +# --------------------------------------------------------------------------- + +FOREIGN_KEYS_SQL = """ +SELECT con.conname AS name, + src.relname AS child_table, + (SELECT array_agg(a.attname ORDER BY u.ord) + FROM unnest(con.conkey) WITH ORDINALITY AS u(attnum, ord) + JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = u.attnum + ) AS child_cols, + tgt.relname AS parent_table, + (SELECT array_agg(a.attname ORDER BY u.ord) + FROM unnest(con.confkey) WITH ORDINALITY AS u(attnum, ord) + JOIN pg_attribute a ON a.attrelid = con.confrelid AND a.attnum = u.attnum + ) AS parent_cols, + con.convalidated AS validated + FROM pg_constraint con + JOIN pg_class src ON src.oid = con.conrelid + JOIN pg_class tgt ON tgt.oid = con.confrelid + JOIN pg_namespace ns ON ns.oid = con.connamespace + WHERE con.contype = 'f' AND ns.nspname = 'public' + ORDER BY src.relname, con.conname +""" + +INVALID_INDEXES_SQL = """ +SELECT c.relname AS index_name, + t.relname AS table_name, + i.indisvalid AS is_valid, + i.indisready AS is_ready, + i.indislive AS is_live + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_class t ON t.oid = i.indrelid + JOIN pg_namespace ns ON ns.oid = c.relnamespace + WHERE ns.nspname = 'public' + AND NOT (i.indisvalid AND i.indisready AND i.indislive) + ORDER BY c.relname +""" + +COLUMNS_SQL = """ +SELECT table_name, column_name, data_type, is_nullable, coalesce(column_default, '') AS def + FROM information_schema.columns + WHERE table_schema = 'public' +""" + +INDEXES_SQL = "SELECT indexname AS name, indexdef AS def FROM pg_indexes WHERE schemaname='public'" + +CONSTRAINTS_SQL = """ +SELECT con.conname AS name, rel.relname AS table_name, + pg_get_constraintdef(con.oid) AS def + FROM pg_constraint con + JOIN pg_class rel ON rel.oid = con.conrelid + JOIN pg_namespace ns ON ns.oid = con.connamespace + WHERE ns.nspname = 'public' +""" + +ENUMS_SQL = """ +SELECT t.typname AS name, + array_agg(e.enumlabel ORDER BY e.enumsortorder) AS labels + FROM pg_type t + JOIN pg_enum e ON e.enumtypid = t.oid + JOIN pg_namespace ns ON ns.oid = t.typnamespace + WHERE ns.nspname = 'public' + GROUP BY t.typname +""" + + +# --------------------------------------------------------------------------- +# Checks +# --------------------------------------------------------------------------- + + +async def check_revision(conn, target: str): + rev = await current_revision(conn) + title = "alembic_version is exactly the target revision" + if rev == target: + return ( + title, + PASS, + f"alembic_version = {rev!r}. NOTE: this proves nothing on its own — see the " + "schema checks below.", + [], + {"current_revision": rev, "target": target}, + ) + if rev is None: + return ( + title, + FAIL, + "no alembic_version row (or no alembic_version table). If `alembic upgrade` " + "just printed a successful chain, this is the silent-rollback signature: the " + "harness never committed.", + [ + "Check the harness, not the data:", + " python scripts/migrate/preflight.py # check 8", + "Then re-run the upgrade and re-run this script.", + ], + {"current_revision": None, "target": target}, + ) + return ( + title, + FAIL, + f"alembic_version = {rev!r}, expected {target!r}.", + [f" python -m alembic upgrade {target}", "then re-run this script."], + {"current_revision": rev, "target": target}, + ) + + +async def check_expected_columns(conn): + title = "Every column 0019/0020/0023 adds exists, with the right type and nullability" + live = { + (r["table_name"], r["column_name"]): r + for r in await fetch_all(conn, COLUMNS_SQL) + } + problems: list[str] = [] + for table, column, dtype, nullable, default in EXPECTED_COLUMNS: + row = live.get((table, column)) + if row is None: + problems.append(f"{table}.{column} is MISSING") + continue + if row["data_type"] != dtype: + problems.append( + f"{table}.{column} type is {row['data_type']!r}, expected {dtype!r}" + ) + live_nullable = row["is_nullable"] == "YES" + if live_nullable != nullable: + problems.append( + f"{table}.{column} is {'NULL' if live_nullable else 'NOT NULL'}able, " + f"expected {'nullable' if nullable else 'NOT NULL'}" + ) + if default is not None and row["def"] != default: + problems.append( + f"{table}.{column} default is {row['def']!r}, expected {default!r}" + ) + if problems: + return ( + title, + FAIL, + "\n".join([f"{len(problems)} column problem(s):"] + [f" {p}" for p in problems]), + ["The schema does not match the migrations. Do NOT let the app start against " + "it. Restore from the backup, or work out which revision really ran:", + " python scripts/migrate/preflight.py --json"], + {"problems": problems}, + ) + return ( + title, + PASS, + f"{len(EXPECTED_COLUMNS)} columns checked (type, nullability, server default).", + [], + {"checked": len(EXPECTED_COLUMNS)}, + ) + + +async def check_expected_tables(conn): + title = "Every table 0020/0022 creates exists" + missing = [t for t in EXPECTED_TABLES if not await table_exists(conn, t)] + if missing: + return ( + title, + FAIL, + f"missing table(s): {', '.join(missing)}", + [" python -m alembic upgrade 0023", "then re-run this script."], + {"missing": missing}, + ) + return (title, PASS, f"{len(EXPECTED_TABLES)} tables present: {', '.join(EXPECTED_TABLES)}.", + [], {"checked": list(EXPECTED_TABLES)}) + + +async def check_expected_indexes(conn): + title = "Every index 0019/0020/0021/0022 creates exists, on the right columns" + live = {r["name"]: r["def"] for r in await fetch_all(conn, INDEXES_SQL)} + problems: list[str] = [] + for name, tail in EXPECTED_INDEXES.items(): + if name not in live: + problems.append(f"{name} is MISSING") + elif tail not in live[name]: + problems.append(f"{name} definition is {live[name]!r}, expected to contain {tail!r}") + if problems: + return ( + title, + FAIL, + "\n".join([f"{len(problems)} index problem(s):"] + [f" {p}" for p in problems]), + ["Recreate the missing index(es) by hand, or restore and re-migrate. The " + "partial index ix_agent_messages_run_slack_ts must keep its " + "WHERE (slack_ts IS NOT NULL) predicate — without it the index is a " + "different, much larger object that no query planner will use the same way."], + {"problems": problems}, + ) + return (title, PASS, f"{len(EXPECTED_INDEXES)} indexes checked, definitions match.", [], + {"checked": len(EXPECTED_INDEXES)}) + + +async def check_expected_constraints(conn): + title = "Constraints 0019/0022 add exist with the right definition" + live = {r["name"]: (r["table_name"], r["def"]) for r in await fetch_all(conn, CONSTRAINTS_SQL)} + problems: list[str] = [] + for name, (table, expect) in EXPECTED_CONSTRAINTS.items(): + got = live.get(name) + if got is None: + problems.append(f"{name} on {table} is MISSING") + elif got[0] != table: + problems.append(f"{name} is on {got[0]!r}, expected {table!r}") + elif expect not in got[1]: + problems.append(f"{name} def is {got[1]!r}, expected to contain {expect!r}") + if problems: + return (title, FAIL, + "\n".join([f"{len(problems)} constraint problem(s):"] + [f" {p}" for p in problems]), + ["Without uq_agent_messages_run_ts the DB-primary write path loses its " + "idempotency key: the flush upserts on (simulation_run_id, message_ts) " + "and will start inserting duplicates instead."], + {"problems": problems}) + return (title, PASS, f"{len(EXPECTED_CONSTRAINTS)} constraints checked.", [], + {"checked": len(EXPECTED_CONSTRAINTS)}) + + +async def check_enums(conn): + """0020 creates pi_dm_direction_enum inline in create_table, with no checkfirst.""" + title = "pi_dm_direction_enum has exactly the expected values" + live = {r["name"]: tuple(r["labels"]) for r in await fetch_all(conn, ENUMS_SQL)} + problems: list[str] = [] + for name, labels in EXPECTED_ENUMS.items(): + got = live.get(name) + if got is None: + problems.append(f"type {name} is MISSING") + elif got != labels: + problems.append(f"type {name} has {list(got)}, expected {list(labels)}") + if problems: + return (title, FAIL, + "\n".join(problems), + ["An enum with extra values means someone ran ALTER TYPE ... ADD VALUE by " + "hand; note that added values cannot be removed, so the type must be " + "recreated:", + " -- inspect first: SELECT DISTINCT direction FROM pi_dm_messages;"], + {"problems": problems, "live": {k: list(v) for k, v in live.items()}}) + return (title, PASS, + ", ".join(f"{k} = {list(v)}" for k, v in EXPECTED_ENUMS.items()) + ".", + [], {"live": {k: list(v) for k, v in live.items() if k in EXPECTED_ENUMS}}) + + +async def check_no_unintended_nulls(conn): + """Catalog says NOT NULL, and the data agrees. + + The catalog check is the real one; the data check exists because it is the only + thing that survives someone doing ``ALTER COLUMN ... DROP NOT NULL`` to make an + insert work. + """ + title = "No unintended NULLs in the columns the migrations declare NOT NULL" + problems: list[str] = [] + counts: dict[str, int] = {} + for table, column in MUST_BE_NON_NULL: + if not await table_exists(conn, table): + problems.append(f"{table} does not exist") + continue + enforced = await fetch_one_value( + conn, + "SELECT attnotnull FROM pg_attribute " + f"WHERE attrelid='{table}'::regclass AND attname=:c", + c=column, + ) + n = int(await fetch_one_value(conn, f'SELECT count(*) FROM "{table}" WHERE "{column}" IS NULL')) + counts[f"{table}.{column}"] = n + if not enforced: + problems.append(f"{table}.{column} is not NOT NULL in the catalog") + if n: + problems.append(f"{table}.{column} has {n:,} NULL row(s)") + if problems: + return (title, FAIL, "\n".join(problems), + ["Find and fix the rows, then re-apply the constraint:", + " UPDATE SET = WHERE IS NULL;", + " ALTER TABLE
ALTER COLUMN SET NOT NULL;"], + {"problems": problems, "null_counts": counts}) + return (title, PASS, f"{len(MUST_BE_NON_NULL)} NOT NULL columns verified in catalog and data.", + [], {"null_counts": counts}) + + +async def check_fk_integrity(conn): + title = "No foreign-key orphans, and every FK is convalidated" + fks = await fetch_all(conn, FOREIGN_KEYS_SQL) + problems: list[str] = [] + orphan_counts: dict[str, int] = {} + for fk in fks: + if not fk["validated"]: + problems.append(f"{fk['name']} on {fk['child_table']} is NOT VALIDATED") + child_cols = list(fk["child_cols"] or []) + parent_cols = list(fk["parent_cols"] or []) + if not child_cols or len(child_cols) != len(parent_cols): + continue + not_null = " AND ".join(f'c."{c}" IS NOT NULL' for c in child_cols) + join = " AND ".join( + f'p."{p}" = c."{c}"' for c, p in zip(child_cols, parent_cols, strict=True) + ) + n = int( + await fetch_one_value( + conn, + f'SELECT count(*) FROM "{fk["child_table"]}" c WHERE {not_null} ' + f'AND NOT EXISTS (SELECT 1 FROM "{fk["parent_table"]}" p WHERE {join})', + ) + ) + if n: + orphan_counts[fk["name"]] = n + problems.append( + f"{fk['name']}: {n:,} row(s) in {fk['child_table']}" + f"({', '.join(child_cols)}) with no {fk['parent_table']} parent" + ) + if problems: + return (title, FAIL, "\n".join(problems), + ["Orphans mean a FK was created NOT VALID, or rows were inserted with " + "triggers disabled. Validate explicitly to see them all:", + " ALTER TABLE VALIDATE CONSTRAINT ;"], + {"foreign_keys": len(fks), "problems": problems, "orphans": orphan_counts}) + return (title, PASS, f"{len(fks)} foreign keys: all convalidated, 0 orphans.", [], + {"foreign_keys": len(fks)}) + + +async def check_index_validity(conn): + title = "No invalid indexes (pg_index.indisvalid / indisready / indislive)" + bad = await fetch_all(conn, INVALID_INDEXES_SQL) + if bad: + return (title, FAIL, + "\n".join( + f" {r['index_name']} on {r['table_name']} " + f"valid={r['is_valid']} ready={r['is_ready']} live={r['is_live']}" + for r in bad + ), + ["An invalid index is not used by the planner and does not enforce " + "uniqueness. Rebuild it:", + " REINDEX INDEX ; -- or DROP and recreate"], + {"invalid": bad}) + return (title, PASS, "every index in public is valid, ready and live.", [], {}) + + +async def check_row_counts(conn, snapshot_path: str | None, allow_growth: bool): + title = "Row counts match the preflight snapshot" + counts = await _pf.snapshot_row_counts(conn) + if not snapshot_path: + return (title, WARN, + f"no --snapshot given; counted {sum(counts.values()):,} rows across " + f"{len(counts)} tables but had nothing to compare against.", + ["Run preflight with --snapshot before the migration, and pass the " + "same path here."], + {"row_counts": counts}) + p = Path(snapshot_path) + if not p.is_file(): + return (title, FAIL, f"snapshot {snapshot_path} does not exist.", + ["The handoff file is the only record of the pre-migration counts. Without " + "it this run cannot show that no rows were lost."], + {"row_counts": counts}) + try: + payload = json.loads(p.read_text()) + except (OSError, ValueError) as exc: + return (title, FAIL, f"snapshot {snapshot_path} is unreadable: {exc}", [], + {"row_counts": counts}) + before = {k: int(v) for k, v in (payload.get("row_counts") or {}).items()} + ok, problems = compare_row_counts( + before, counts, allow_growth=allow_growth, expected_new=CHAIN_CREATED_TABLES + ) + data = {"row_counts": counts, "snapshot_row_counts": before, "problems": problems} + if ok: + return (title, PASS, + f"{len(before)} tables, {sum(before.values()):,} rows, identical before and " + "after.", [], data) + return (title, FAIL, + "\n".join([f"{len(problems)} row-count problem(s):"] + [f" {x}" for x in problems]), + ["Row loss is not something a migration in this chain can cause, so treat it as " + "either the wrong snapshot file or a concurrent writer/deleter. Compare against " + "the backup before doing anything else.", + "Growth alone (not loss) can be accepted with --allow-row-growth, but only if " + "you know a writer was live."], + data) + + +async def check_orm_can_query(conn_url: str): + """Import the real models at HEAD and run a real query per mapper. + + A schema that satisfies every catalog assertion above can still be unusable: the ORM + selects every mapped column by name, so one missing column breaks every query against + that model. This is the check that speaks for the application rather than the schema. + """ + title = "The ORM at HEAD can query every model" + from sqlalchemy import select + from sqlalchemy.ext.asyncio import create_async_engine + + import src.models # noqa: F401 + from src.database import Base + + mappers = sorted(Base.registry.mappers, key=lambda m: m.class_.__name__) + engine = create_async_engine(conn_url, isolation_level="AUTOCOMMIT", pool_pre_ping=False) + failures: list[str] = [] + checked: list[str] = [] + try: + async with engine.connect() as c: + for mapper in mappers: + model = mapper.class_ + try: + await c.execute(select(model).limit(1)) + checked.append(model.__name__) + except Exception as exc: # noqa: BLE001 - the message is the finding + first = str(exc).strip().splitlines()[0] + failures.append(f"{model.__name__} ({mapper.local_table}): {first}") + finally: + await engine.dispose() + if failures: + return (title, FAIL, + "\n".join([f"{len(failures)} model(s) cannot be queried:"] + + [f" {f}" for f in failures]), + ["The application will fail on its first request against these models. Do " + "not start it. Fix the schema or restore."], + {"failures": failures, "ok": checked}) + return (title, PASS, f"{len(checked)} models each returned from a real SELECT ... LIMIT 1.", + [], {"ok": checked}) + + +async def check_orm_drift(conn_url: str): + """alembic autogenerate diff, filtered to "the DB is missing what the ORM needs".""" + title = "No ORM drift (nothing the models require is absent from the database)" + from alembic.autogenerate import compare_metadata + from alembic.migration import MigrationContext + from sqlalchemy.ext.asyncio import create_async_engine + + import src.models # noqa: F401 + from src.database import Base + + def _diff(sync_conn): + return compare_metadata(MigrationContext.configure(sync_conn), Base.metadata) + + engine = create_async_engine(conn_url, isolation_level="AUTOCOMMIT", pool_pre_ping=False) + try: + async with engine.connect() as c: + raw = await c.run_sync(_diff) + finally: + await engine.dispose() + + failures: list[str] = [] + ignored = 0 + unknown: list[str] = [] + for entry in raw: + items = entry if isinstance(entry, list) else [entry] + for item in items: + op = item[0] if isinstance(item, (tuple, list)) else str(item) + if op in DRIFT_FAIL_OPS: + failures.append(f"{op}: {str(item)[:180]}") + elif op in DRIFT_IGNORED_OPS: + ignored += 1 + else: + unknown.append(f"{op}: {str(item)[:180]}") + data = {"failures": failures, "ignored": ignored, "unclassified": unknown} + if failures: + return (title, FAIL, + "\n".join([f"{len(failures)} drift finding(s) the models cannot tolerate:"] + + [f" {f}" for f in failures]), + ["Each add_column/add_index/add_table means the database lacks something a " + "model declares; modify_nullable/modify_type means it has it with the " + "wrong shape. Restore, or apply the missing DDL and re-run."], + data) + detail = ( + f"0 findings that matter; {ignored} pre-existing differences ignored (indexes and " + "inline unique constraints created before 0018 that the models never declare, plus " + "spurious add_table_comment entries — measured: 25 on a correctly migrated database)." + ) + if unknown: + return (title, WARN, detail + f" {len(unknown)} unclassified op(s): {unknown}", [], data) + return (title, PASS, detail, [], data) + + +async def run_postflight(args) -> Report: + # warn_exit_code=0: postflight's contract is 0 = verified / 1 = failed, so a WARN is + # reported loudly but does not fail the run. + report = Report("postflight", warn_exit_code=0) + url = resolve_database_url(args.database_url) + report.extra["database_url"] = redact_url(url) + report.extra["target"] = args.target + + engine, conn = await open_connection(url, args.statement_timeout_ms) + try: + # Every check is fenced: a verification script that dies with a traceback has + # verified nothing, and reads to the operator as "the tool is broken" rather + # than "the migration is broken". + await report.add_guarded( + "alembic_version is exactly the target revision", + lambda: check_revision(conn, args.target), + ) + + async def _scripts(): + return await check_alembic_scripts(await current_revision(conn), args.target) + + await report.add_guarded("Exactly one alembic head, no duplicate revision ids", _scripts) + await report.add_guarded( + "Every table 0020/0022 creates exists", lambda: check_expected_tables(conn) + ) + await report.add_guarded( + "Every column 0019/0020/0023 adds exists, with the right type and nullability", + lambda: check_expected_columns(conn), + ) + await report.add_guarded( + "Every index 0019/0020/0021/0022 creates exists, on the right columns", + lambda: check_expected_indexes(conn), + ) + await report.add_guarded( + "Constraints 0019/0022 add exist with the right definition", + lambda: check_expected_constraints(conn), + ) + await report.add_guarded( + "pi_dm_direction_enum has exactly the expected values", lambda: check_enums(conn) + ) + await report.add_guarded( + "No invalid indexes (pg_index.indisvalid / indisready / indislive)", + lambda: check_index_validity(conn), + ) + await report.add_guarded( + "No unintended NULLs in the columns the migrations declare NOT NULL", + lambda: check_no_unintended_nulls(conn), + ) + await report.add_guarded( + "No foreign-key orphans, and every FK is convalidated", + lambda: check_fk_integrity(conn), + ) + await report.add_guarded( + "Row counts match the preflight snapshot", + lambda: check_row_counts(conn, args.snapshot, args.allow_row_growth), + ) + finally: + await conn.close() + await engine.dispose() + + # These open their own engines: the ORM checks must run through SQLAlchemy's own + # machinery, not this script's raw connection. + await report.add_guarded( + "No ORM drift (nothing the models require is absent from the database)", + lambda: check_orm_drift(url), + ) + await report.add_guarded( + "The ORM at HEAD can query every model", lambda: check_orm_can_query(url) + ) + return report + + +def build_parser(): + import argparse + + ap = argparse.ArgumentParser( + prog="postflight", + description="Post-migration verification. Exit 0 = verified, 1 = verification failed.", + ) + add_common_arguments(ap) + # Re-document --target: it moves the revision assertion, NOT the schema expectations. + for action in ap._actions: # noqa: SLF001 - argparse offers no public way to do this + if action.dest == "target": + action.help = ( + f"Revision alembic_version must equal (default {DEFAULT_TARGET}). NOTE: the " + "schema, index, constraint and enum expectations describe 0023 and only " + "0023, so --target 0019 will match the stamp and then correctly report " + "everything 0020-0023 has not yet created." + ) + ap.add_argument("--snapshot", default=None, help="Row-count snapshot written by preflight") + ap.add_argument( + "--allow-row-growth", + action="store_true", + help="Treat a table that GREW as acceptable (row loss always fails).", + ) + return ap + + +def main(argv: list[str] | None = None) -> int: + import asyncio + + args = build_parser().parse_args(argv) + report = asyncio.run(run_postflight(args)) + return _pf.emit(report, args) + + +if __name__ == "__main__": + os.environ.setdefault("PYTHONWARNINGS", "ignore") + raise SystemExit(main()) diff --git a/scripts/migrate/preflight.py b/scripts/migrate/preflight.py new file mode 100644 index 0000000..40c242a --- /dev/null +++ b/scripts/migrate/preflight.py @@ -0,0 +1,1916 @@ +#!/usr/bin/env python3 +"""Pre-migration safety gate for the 0018/0019 -> 0023 upgrade. + +Run this BEFORE `alembic upgrade`, against the database you are about to migrate. +It answers one question: *will this migration succeed, and what will it cost?* + + docker compose exec -T -e DATABASE_URL=... app python scripts/migrate/preflight.py + +Exit codes (contract; other tooling depends on these): + + 0 safe to migrate — every check PASSed + 1 BLOCKED, do not migrate — at least one check BLOCKed + 2 warnings only — operator judgement required + +Every non-PASS item prints the exact remediation command or SQL. Nothing here +writes to the database: the connection runs in AUTOCOMMIT so that preflight can +never itself become the open transaction that stalls the migration. + +Why each check exists is documented at the check itself. The three that are easy +to underestimate: + + * Check 4 (duplicate ``(simulation_run_id, message_ts)``) is the hard blocker. + Migration 0019 adds ``uq_agent_messages_run_ts``; Postgres reports only ONE + duplicated key per failed index build, so the migration must be re-run once + per duplicate group unless you enumerate them all up front. Check 4 does that + in a single pass. + * Check 3 exists because revision id ``0019`` is ambiguous in this repo's + history. ``alembic/versions/0019_add_cohorts.py`` on the ``cohort-agent-isolation`` + branch (commit b00b0e6) also claimed ``revision = "0019"``, revising 0018. A + database migrated from that branch is stamped ``0019`` while the *content* + columns 0019 adds were never applied — and the chain then dies at 0022 with + ``relation "cohorts" already exists``. + * Check 8 checks the migration *harness*, not the data. If ``alembic/env.py`` + emits any SQL on the connection before ``context.begin_transaction()``, the + connection autobegins, ``MigrationContext.__init__`` sets + ``_in_external_transaction = True``, ``begin_transaction()`` degrades to a + ``nullcontext()`` and alembic never commits — so ``alembic upgrade`` logs a + full successful chain, exits 0, and applies NOTHING. +""" + +from __future__ import annotations + +import argparse +import ast +import gzip +import json +import os +import re +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + # `python scripts/migrate/preflight.py` puts scripts/migrate/ on sys.path, NOT the + # repo root, so a bare `import src...` would resolve to the STALE copy of src/ that + # the Dockerfile's `pip install .` baked into site-packages (verified: it predates + # src/models/cohort.py entirely). Put the repo root first so /app/src wins. + sys.path.insert(0, str(REPO_ROOT)) + +# --------------------------------------------------------------------------- +# Contract constants +# --------------------------------------------------------------------------- + +PASS = "PASS" +WARN = "WARN" +BLOCK = "BLOCK" + +EXIT_OK = 0 +EXIT_BLOCKED = 1 +EXIT_WARN = 2 + +DEFAULT_TARGET = "0024" +#: 0023 is supported because it is where a deployment that already took the cohort +#: migration sits. org1 is at 0018 (see docs/production-migration.md); do not read +#: this tuple as a statement about any one deployment's current stamp. +#: +#: 0020 and 0021 are here because origin/main's own alembic head is 0021 (PR19). A +#: deployment that tracks main is therefore stamped 0021, and the first version of this +#: list — ("0018", "0019") — hard-BLOCKED exactly that state. The framing that produced +#: it ("migrate from 0018 or 0019") described where production was at the time, not where +#: main is. +#: +#: Starting at 0020/0021 is strictly safer than starting at 0018: uq_agent_messages_run_ts +#: already exists, so duplicates cannot be present and there is no 0019 index build to +#: wait on. All that remains is 0022 (three empty tables) and 0023 (three columns on the +#: small researcher_profiles). +SUPPORTED_START_REVISIONS = ("0018", "0019", "0020", "0021", "0023") + +#: Start revisions at which migration 0019 has already run, so the expensive +#: ACCESS EXCLUSIVE index build on agent_messages is behind us. +POST_0019_STARTS = ("0020", "0021") + +#: Tables whose row counts are snapshotted for postflight. Empty = every user table. +SNAPSHOT_SCHEMA = "public" + +# --------------------------------------------------------------------------- +# Sizing calibration — measured, not guessed. +# --------------------------------------------------------------------------- +# Method: build a fixture at 0018, bulk-load N synthetic agent_messages rows, then +# run the 15-statement 0019+0021 DDL block for agent_messages inside ONE transaction +# in psql with \timing on, and sum the statement times. postgres:15, the compose +# postgres container, on this developer's machine, 2026-08-04: +# +# rows DDL block total post-migration total relation size +# 10,011 112.0 ms 3,608 kB (from 2,016 kB, +79%) +# 100,011 747.2 ms 34 MB (from 19 MB, +79%) +# 1,000,011 7,901.8 ms 339 MB (from 188 MB, +80%) +# +# Cross-check via wall clock of `alembic upgrade head` on the same three fixtures: +# 1.77 s / 2.21 s / 8.37 s against a 1.43 s measured no-op baseline (docker exec + +# interpreter start + connect + version probe), i.e. 0.34 / 0.78 / 6.94 s of work. +# Both methods agree to ~10%. +# +# Least squares over the three DDL-block points gives ~0.0079 ms/row with a ~33 ms +# intercept; rounded to the constants below. These are for an idle server with a warm +# cache, which is the FLOOR. CONTENTION_FACTOR is the upper bound quoted to the +# operator: a production server is doing other work and the index build competes for +# I/O and maintenance_work_mem. +LOCK_WINDOW_FIXED_MS = 50.0 +LOCK_WINDOW_PER_ROW_MS = 0.0080 +LOCK_WINDOW_CONTENTION_FACTOR = 3.0 +#: Beyond this, quote the estimate as an extrapolation rather than a measurement. +LOCK_WINDOW_CALIBRATED_MAX_ROWS = 1_000_000 +#: Upper-bound lock window above which the operator should schedule a window. +LOCK_WINDOW_WARN_MS = 10_000.0 +#: 0019+0021 add four indexes to agent_messages. Measured +79%/+79%/+80% of the +#: pre-migration total relation size at the three scales above. +INDEX_GROWTH_FRACTION = 0.80 +#: Above this much *new* index data, tell the operator to check free space by hand; +#: preflight cannot see the filesystem from inside Postgres. +INDEX_GROWTH_WARN_BYTES = 1 << 30 # 1 GiB + +# --------------------------------------------------------------------------- +# Backup thresholds +# --------------------------------------------------------------------------- +DEFAULT_BACKUP_MAX_AGE_HOURS = 24.0 +#: A 0-byte or truncated file. Measured: the smallest *real* dump of this schema +#: (empty database, gzipped) is 6,914 bytes; a schema-only dump is ~38 kB and is +#: essentially CONSTANT regardless of data volume, which is why size alone can never +#: prove a dump carries data — hence the data-section scan below. +DEFAULT_BACKUP_MIN_BYTES = 1024 +#: How much of a text/gzip dump to scan for data sections. Bounded so preflight +#: cannot be made slow by a huge dump. +BACKUP_SCAN_BYTES = 64 << 20 # 64 MiB of decompressed text +#: Default places to look when --backup-path is not given. +DEFAULT_BACKUP_DIRS = ("backups", "data/backups", "/backups", "/var/backups/copi") +BACKUP_GLOBS = ("*.sql", "*.sql.gz", "*.dump", "*.dmp", "*.pgdump", "*.custom", "*.bak") + +# --------------------------------------------------------------------------- +# What the migration chain CREATES, per revision. Derived by reading 0019-0023; +# tests/unit/test_migration_checks.py re-derives this from the migration files and +# asserts it still matches, so it cannot silently drift. +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class PlannedObject: + """A database object a migration will CREATE (so it must not already exist).""" + + revision: str + kind: str # 'table' | 'column' | 'index' | 'constraint' | 'type' + name: str + table: str | None = None + + +PLANNED_OBJECTS: tuple[PlannedObject, ...] = ( + # 0019_agent_message_content + PlannedObject("0019", "column", "content", "agent_messages"), + PlannedObject("0019", "column", "sender_name", "agent_messages"), + PlannedObject("0019", "column", "is_bot", "agent_messages"), + PlannedObject("0019", "column", "posted_at", "agent_messages"), + PlannedObject("0019", "column", "slack_ts", "agent_messages"), + PlannedObject("0019", "column", "slack_channel_id", "agent_messages"), + PlannedObject("0019", "column", "slack_thread_ts", "agent_messages"), + PlannedObject("0019", "constraint", "uq_agent_messages_run_ts", "agent_messages"), + PlannedObject("0019", "index", "ix_agent_messages_run_posted", "agent_messages"), + PlannedObject("0019", "index", "ix_agent_messages_run_channel_posted", "agent_messages"), + PlannedObject("0019", "index", "ix_agent_messages_run_slack_ts", "agent_messages"), + # 0020_pi_dm_messages + PlannedObject("0020", "table", "pi_dm_messages"), + PlannedObject("0020", "type", "pi_dm_direction_enum"), + PlannedObject("0020", "index", "ix_pi_dm_run_agent_posted", "pi_dm_messages"), + PlannedObject("0020", "index", "ix_pi_dm_run_direction_posted", "pi_dm_messages"), + # 0021_inbox_cursor_created_at_indexes + PlannedObject("0021", "index", "ix_agent_messages_run_created", "agent_messages"), + PlannedObject("0021", "index", "ix_pi_dm_run_direction_created", "pi_dm_messages"), + # 0022_add_cohorts + PlannedObject("0022", "table", "cohorts"), + PlannedObject("0022", "table", "cohort_memberships"), + PlannedObject("0022", "table", "cohort_audit_events"), + PlannedObject("0022", "constraint", "uq_cohort_membership_cohort_agent", "cohort_memberships"), + PlannedObject("0022", "index", "ix_cohort_memberships_cohort_id", "cohort_memberships"), + PlannedObject("0022", "index", "ix_cohort_memberships_agent_id", "cohort_memberships"), + PlannedObject("0022", "index", "ix_cohort_audit_events_cohort_id", "cohort_audit_events"), + PlannedObject("0022", "index", "ix_cohort_audit_events_created_at", "cohort_audit_events"), + # 0023_profile_synthesis_provenance + PlannedObject("0023", "column", "synthesis_validated", "researcher_profiles"), + PlannedObject("0023", "column", "evidence_pmid_count", "researcher_profiles"), + PlannedObject("0023", "column", "evidence_pub_count", "researcher_profiles"), + # 0024_add_agent_role + PlannedObject("0024", "column", "role", "agents"), +) + +REVISION_ORDER = ("0018", "0019", "0020", "0021", "0022", "0023", "0024") + + +def planned_objects_between(current: str, target: str) -> tuple[PlannedObject, ...]: + """Objects created by the revisions that will actually run for current -> target. + + A revision already applied cannot collide with itself, so its objects are excluded: + at 0019 the content columns and ``uq_agent_messages_run_ts`` already exist and that + is correct, not a collision. + """ + try: + lo = REVISION_ORDER.index(current) + except ValueError: + lo = 0 + try: + hi = REVISION_ORDER.index(target) + except ValueError: + hi = len(REVISION_ORDER) - 1 + pending = set(REVISION_ORDER[lo + 1 : hi + 1]) + return tuple(o for o in PLANNED_OBJECTS if o.revision in pending) + + +# --------------------------------------------------------------------------- +# SQL — kept as module constants so the unit tests can pin them. +# --------------------------------------------------------------------------- + +#: Every duplicate group in ONE pass, with the row ids. NULL message_ts is excluded +#: because Postgres UNIQUE treats NULLs as distinct (no NULLS NOT DISTINCT here), so +#: many NULL-ts rows in one run are legal — verified: three such rows coexist with +#: uq_agent_messages_run_ts, and the remediation that NULLs the extras migrates clean. +DUPLICATE_GROUPS_SQL = """ +SELECT simulation_run_id::text AS run_id, + message_ts, + count(*) AS n, + array_agg(id::text ORDER BY created_at, id) AS ids + FROM agent_messages + WHERE message_ts IS NOT NULL + GROUP BY simulation_run_id, message_ts +HAVING count(*) > 1 + ORDER BY count(*) DESC, message_ts +""" + +#: Preferred remediation: the extra rows are duplicate ingestions of one message, and +#: (run, ts) is an idempotency key. Verified end to end: applying this to a fixture +#: with three duplicate groups (sizes 3, 2, 2) let `alembic upgrade head` reach 0023. +DEDUPE_DELETE_SQL = """\ +-- Remediation A (preferred): keep the earliest row of each duplicate group. +BEGIN; +WITH ranked AS ( + SELECT id, row_number() OVER ( + PARTITION BY simulation_run_id, message_ts + ORDER BY created_at, id) AS rn + FROM agent_messages + WHERE message_ts IS NOT NULL +) +DELETE FROM agent_messages WHERE id IN (SELECT id FROM ranked WHERE rn > 1); +COMMIT;""" + +#: Alternative that deletes nothing. NULL is exempt from the unique constraint, so the +#: extra rows survive with their canonical id cleared. Verified end to end: 18 rows in, +#: 18 rows out, 11 keeping a message_ts, upgrade reached 0023. +DEDUPE_NULL_SQL = """\ +-- Remediation B (loses no rows; the extras lose their canonical message_ts, so they +-- can no longer be matched to a Slack message or upserted idempotently). +BEGIN; +WITH ranked AS ( + SELECT id, row_number() OVER ( + PARTITION BY simulation_run_id, message_ts + ORDER BY created_at, id) AS rn + FROM agent_messages + WHERE message_ts IS NOT NULL +) +UPDATE agent_messages SET message_ts = NULL WHERE id IN (SELECT id FROM ranked WHERE rn > 1); +COMMIT;""" + +BLOCKING_SESSIONS_SQL = """ +SELECT a.pid, + a.state, + a.usename, + a.application_name, + coalesce(extract(epoch FROM (now() - a.xact_start)), 0)::float AS xact_age_s, + coalesce(extract(epoch FROM (now() - a.query_start)), 0)::float AS query_age_s, + coalesce(a.query, '') AS query, + EXISTS ( + SELECT 1 FROM pg_locks l + WHERE l.pid = a.pid + -- to_regclass, not 'agent_messages'::regclass: the cast raises + -- UndefinedTable on a database where the table does not exist yet, which + -- would make preflight crash instead of reporting. NULL compares false. + AND l.relation = to_regclass('public.agent_messages') + ) AS holds_agent_messages_lock + FROM pg_stat_activity a + WHERE a.datname = current_database() + AND a.pid <> pg_backend_pid() + AND a.backend_type = 'client backend' + -- Exclude our own tooling by name as well as by pid, so a concurrently-running + -- preflight/postflight is not reported as a blocker of the migration it is checking. + AND coalesce(a.application_name, '') <> :app_name + AND (a.xact_start IS NOT NULL OR a.state <> 'idle') + ORDER BY a.xact_start NULLS LAST +""" + +#: Set on our own connections and excluded from BLOCKING_SESSIONS_SQL. +APPLICATION_NAME = "copi_migration_check" + +ROW_COUNT_SQL = """ +SELECT c.relname AS table_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = :schema + AND c.relkind = 'r' + ORDER BY c.relname +""" + +# --------------------------------------------------------------------------- +# Pure helpers (no database, no I/O) — unit tested in tests/unit/test_migration_checks.py +# --------------------------------------------------------------------------- + + +def exit_code_for(statuses: list[str]) -> int: + """BLOCK dominates WARN dominates PASS.""" + if BLOCK in statuses: + return EXIT_BLOCKED + if WARN in statuses: + return EXIT_WARN + return EXIT_OK + + +def worst_status(statuses: list[str]) -> str: + if BLOCK in statuses: + return BLOCK + if WARN in statuses: + return WARN + return PASS + + +def worst_status_exit(statuses: list[str]) -> int: + """The exit code the statuses *mean*, ignoring any per-script remapping of WARN.""" + return exit_code_for(statuses) + + +def estimate_lock_window_ms(rows: int) -> tuple[float, float]: + """(floor, ceiling) milliseconds of ACCESS EXCLUSIVE on agent_messages. + + The whole 0019..0023 chain runs in ONE transaction (env.py does not pass + ``transaction_per_migration``), so the lock 0019's first ``ADD COLUMN`` takes is + held until the final commit: the lock window is the chain, not the index build. + """ + floor_ms = LOCK_WINDOW_FIXED_MS + LOCK_WINDOW_PER_ROW_MS * max(rows, 0) + return floor_ms, floor_ms * LOCK_WINDOW_CONTENTION_FACTOR + + +def sizing_status(rows: int, ceiling_ms: float) -> tuple[str, str]: + """PASS for a sub-10s window; WARN once a maintenance window is warranted.""" + if rows > LOCK_WINDOW_CALIBRATED_MAX_ROWS: + return ( + WARN, + f"{rows:,} rows is beyond the calibrated range (<= " + f"{LOCK_WINDOW_CALIBRATED_MAX_ROWS:,}); the estimate is an extrapolation.", + ) + if ceiling_ms > LOCK_WINDOW_WARN_MS: + return ( + WARN, + f"worst-case lock window {ceiling_ms / 1000:.1f}s exceeds " + f"{LOCK_WINDOW_WARN_MS / 1000:.0f}s — schedule a window, do not migrate hot.", + ) + return PASS, f"worst-case lock window {ceiling_ms / 1000:.1f}s." + + +def revision_status(current: str | None, target: str) -> tuple[str, str]: + """Classify the DB's stamped revision against the supported starting points.""" + if current is None: + return ( + BLOCK, + "no alembic_version row (or no alembic_version table): this database has " + "never been stamped, so alembic would replay the chain from 0001 over " + "whatever schema is already there.", + ) + if current == target: + return PASS, f"already at the target revision {target}; migration is a no-op." + if current in SUPPORTED_START_REVISIONS: + return PASS, f"at {current}, a supported starting point." + return ( + BLOCK, + f"at {current}, which is not a supported starting point " + f"({', '.join(SUPPORTED_START_REVISIONS)}) nor the target {target}.", + ) + + +def resolve_lock_timeout_ms(env: dict[str, str], env_py_source: str | None) -> tuple[str, str]: + """What lock_timeout the migration will actually run with, and where it came from.""" + if "ALEMBIC_LOCK_TIMEOUT_MS" in env: + return env["ALEMBIC_LOCK_TIMEOUT_MS"], "ALEMBIC_LOCK_TIMEOUT_MS in the environment" + if env_py_source: + m = re.search( + r"""ALEMBIC_LOCK_TIMEOUT_MS["']?\s*,\s*["'](\d+)["']""", env_py_source + ) + if m: + return m.group(1), "alembic/env.py default" + if "lock_timeout" not in env_py_source: + return "0", "alembic/env.py sets no lock_timeout (Postgres default: wait forever)" + return "unknown", "could not determine" + + +def harness_findings(env_py_source: str) -> list[str]: + """Statements in ``do_run_migrations`` that autobegin a transaction too early. + + If the connection is already in a transaction when ``context.configure()`` builds + the MigrationContext, ``_in_external_transaction`` is set and + ``begin_transaction()`` returns a ``nullcontext()``. Alembic then assumes the caller + owns the transaction; env.py's caller (``run_async_migrations``) exits its + ``connect()`` block without committing, so the DDL is rolled back and + ``alembic upgrade`` still exits 0 with a full "Running upgrade" log. + + Verified on this repo: with ``lock_timeout`` set this way, ``alembic upgrade 0018`` + reported nine successful revisions and left the database with zero tables; with + ``ALEMBIC_LOCK_TIMEOUT_MS=0`` (which skips the statement) the same command left 25 + tables and version 0018. + """ + autobegin_methods = {"execute", "exec_driver_sql", "scalar", "scalars", "begin"} + try: + tree = ast.parse(env_py_source) + except SyntaxError as exc: # pragma: no cover - defensive + return [f"could not parse alembic/env.py: {exc}"] + + findings: list[str] = [] + for func in ast.walk(tree): + if not isinstance(func, ast.FunctionDef) or func.name != "do_run_migrations": + continue + conn_names = {a.arg for a in func.args.args} + for node in ast.walk(func): + if not isinstance(node, ast.Call): + continue + fn = node.func + if not isinstance(fn, ast.Attribute): + continue + # Stop looking once we reach the demarcation call itself. + if fn.attr in {"configure", "begin_transaction"}: + continue + if fn.attr in autobegin_methods and isinstance(fn.value, ast.Name): + if fn.value.id in conn_names: + findings.append( + f"alembic/env.py:{node.lineno} calls " + f"{fn.value.id}.{fn.attr}(...) inside do_run_migrations before " + "context.begin_transaction(), which autobegins a transaction and " + "makes alembic skip its own commit" + ) + return findings + + +def legacy_inventory_status(recoverable: int, unrecoverable: int) -> tuple[str, str]: + """Rows that end up with ``content = ''``: Slack-recoverable vs gone for good.""" + if recoverable == 0 and unrecoverable == 0: + return PASS, "no rows will be left with an empty content column." + parts = [] + if recoverable: + parts.append(f"{recoverable:,} Slack-recoverable (channel_id NOT LIKE 'local:%')") + if unrecoverable: + parts.append(f"{unrecoverable:,} PERMANENTLY UNRECOVERABLE (channel_id LIKE 'local:%')") + return WARN, "; ".join(parts) + "." + + +#: An `active` session holding a transaction younger than this will release its locks on +#: its own before it matters; older than this and it is indistinguishable from a stuck +#: one. Idle-in-transaction is BLOCKed at any age, because nothing will end it. +DEFAULT_MAX_TOLERABLE_XACT_AGE_S = 5.0 + + +def blocking_sessions_status( + sessions: list[dict], max_xact_age_s: float = DEFAULT_MAX_TOLERABLE_XACT_AGE_S +) -> tuple[str, str]: + """BLOCK on idle-in-transaction (any age) or a long open transaction; WARN otherwise. + + An idle-in-transaction reader is enough to stall the whole table: verified — one + ``BEGIN; SELECT count(*) FROM agent_messages;`` left idle made + ``alembic upgrade head`` from 0018 fail on ``lock_timeout`` after 3s, and while the + ACCESS EXCLUSIVE request sat ungranted in the queue a brand-new + ``SELECT count(*)`` from a third session timed out too, because a pending + ACCESS EXCLUSIVE queues AHEAD of new readers. + """ + in_txn = [s for s in sessions if (s.get("xact_age_s") or 0) > 0] + idle_in_txn = [s for s in in_txn if str(s.get("state", "")).startswith("idle in transaction")] + if idle_in_txn: + return ( + BLOCK, + f"{len(idle_in_txn)} session(s) idle in transaction. A pending " + "ACCESS EXCLUSIVE request queues ahead of new readers, so this migration " + "would stall every query on agent_messages until the idle transaction ends.", + ) + long_txn = [s for s in in_txn if (s.get("xact_age_s") or 0) > max_xact_age_s] + if long_txn: + return ( + BLOCK, + f"{len(long_txn)} session(s) with a transaction open longer than " + f"{max_xact_age_s:.0f}s. Any of them can hold a lock that conflicts with " + "0019's ACCESS EXCLUSIVE on agent_messages.", + ) + if in_txn: + return ( + WARN, + f"{len(in_txn)} session(s) with a transaction open for under " + f"{max_xact_age_s:.0f}s. Short enough to release on its own, but it means the " + "database is still being used; stop the writers before migrating.", + ) + if sessions: + return ( + WARN, + f"{len(sessions)} active session(s) with no open transaction. They will be " + "blocked (not blocking) for the duration of the lock window; a writer that " + "retries is fine, one that raises is not.", + ) + return PASS, "no other client sessions on this database." + + +@dataclass +class BackupFacts: + """Everything the backup check needs to decide, gathered by ``inspect_backup``.""" + + path: str | None = None + exists: bool = False + size_bytes: int = 0 + age_hours: float | None = None + fmt: str = "unknown" # 'plain' | 'gzip' | 'custom' | 'unknown' + scanned: bool = False + has_agent_messages_ddl: bool = False + has_agent_messages_data: bool = False + read_error: str | None = None + override_reason: str | None = None + live_agent_messages_rows: int = 0 + + +def evaluate_backup( + facts: BackupFacts, + max_age_hours: float = DEFAULT_BACKUP_MAX_AGE_HOURS, + min_bytes: int = DEFAULT_BACKUP_MIN_BYTES, +) -> tuple[str, list[str]]: + """BLOCK unless a recent, data-bearing dump is demonstrably present. + + This matters more than it looks: 0019's downgrade DROPs the content columns, so a + rollback past 0019 destroys every message body written after the cutover. The dump + is the only way back. + + Size alone cannot establish that a dump carries data. Measured on this schema: a + ``--schema-only`` dump is ~38 kB at 100k rows AND at 1M rows, while a full gzipped + dump is 7-9% of ``pg_database_size``. So the check looks for an actual data section. + """ + notes: list[str] = [] + if facts.override_reason: + return WARN, [ + "backup NOT verified by preflight; overridden with " + f"--backup-verified-elsewhere={facts.override_reason!r}. " + "Rollback past 0019 destroys agent_messages.content — be certain." + ] + if not facts.exists: + return BLOCK, [ + "no backup found. Rollback past 0019 DROPs agent_messages.content, " + "sender_name, is_bot and posted_at, so there is no way back without one.", + "Take one and re-run:", + " docker compose exec -T postgres pg_dump -U copi -d copi | gzip " + "> backups/copi_$(date +%Y%m%dT%H%M%S).sql.gz", + "then pass --backup-path backups/.", + ] + if facts.read_error: + return BLOCK, [f"backup at {facts.path} could not be read: {facts.read_error}"] + if facts.size_bytes < min_bytes: + return BLOCK, [ + f"backup at {facts.path} is {facts.size_bytes:,} bytes, under the " + f"{min_bytes:,}-byte floor — truncated or empty." + ] + if facts.age_hours is not None and facts.age_hours > max_age_hours: + return BLOCK, [ + f"backup at {facts.path} is {facts.age_hours:.1f}h old, older than the " + f"{max_age_hours:.0f}h threshold. Every message written since is not in it.", + " docker compose exec -T postgres pg_dump -U copi -d copi | gzip " + "> backups/copi_$(date +%Y%m%dT%H%M%S).sql.gz", + ] + if facts.fmt == "custom": + return WARN, [ + f"backup at {facts.path} is a pg_dump custom-format archive. Neither " + "pg_restore nor psql is installed in the app image, so preflight cannot " + "confirm it carries data. Verify by hand on the postgres container:", + " docker compose exec -T postgres pg_restore -l /path/to/dump " + "| grep -c 'TABLE DATA'", + ] + if facts.fmt == "unknown": + return BLOCK, [ + f"backup at {facts.path} is not a recognisable pg_dump output (not plain " + "SQL, not gzip, no PGDMP magic). Point --backup-path at a real dump." + ] + if not facts.has_agent_messages_ddl: + return BLOCK, [ + f"backup at {facts.path} contains no agent_messages definition at all — it " + "is not a dump of this database." + ] + if facts.live_agent_messages_rows > 0 and not facts.has_agent_messages_data: + return BLOCK, [ + f"backup at {facts.path} defines agent_messages but contains NO data section " + f"for it, while the live table holds {facts.live_agent_messages_rows:,} rows. " + "This is a --schema-only dump; restoring it would lose every message.", + " docker compose exec -T postgres pg_dump -U copi -d copi | gzip " + "> backups/copi_$(date +%Y%m%dT%H%M%S).sql.gz", + ] + notes.append( + f"{facts.path} — {facts.size_bytes:,} bytes, " + f"{'age unknown' if facts.age_hours is None else f'{facts.age_hours:.1f}h old'}, " + f"format {facts.fmt}, agent_messages data section present." + ) + return PASS, notes + + +def compare_row_counts( + before: dict[str, int], + after: dict[str, int], + allow_growth: bool = False, + expected_new: tuple[str, ...] | frozenset[str] = (), +) -> tuple[bool, list[str]]: + """Compare a preflight snapshot against a postflight count. Shared by both scripts. + + Shrinkage is always a failure. Growth is a failure unless ``allow_growth``, because + the migration itself inserts no rows: if a count went up, a writer was live during + the migration and the lock-window analysis was wrong. + + ``expected_new`` names the tables the chain CREATES (0020's pi_dm_messages, 0022's + three cohort tables). Those are absent from the preflight snapshot by construction, + so flagging them would be crying wolf — but a table appearing that the chain does + not create is still a finding. + """ + problems: list[str] = [] + expected_new = frozenset(expected_new) + for table in sorted(set(before) | set(after)): + b = before.get(table) + a = after.get(table) + if b is None: + if table in expected_new: + continue + problems.append(f"{table}: table did not exist before the migration, now {a:,} rows") + continue + if a is None: + problems.append(f"{table}: existed before with {b:,} rows, now MISSING") + continue + if a < b: + problems.append(f"{table}: {b:,} rows before, {a:,} after — {b - a:,} rows LOST") + elif a > b and not allow_growth: + problems.append( + f"{table}: {b:,} rows before, {a:,} after — grew by {a - b:,}; the " + "migration inserts nothing, so a writer was live" + ) + return (not problems), problems + + +def normalize_async_url(url: str) -> str: + """Force the asyncpg driver, whatever dialect spelling the caller used.""" + if url.startswith("postgresql+asyncpg://"): + return url + for prefix in ("postgresql+psycopg2://", "postgresql+psycopg://", "postgresql://", "postgres://"): + if url.startswith(prefix): + return "postgresql+asyncpg://" + url[len(prefix) :] + return url + + +def redact_url(url: str) -> str: + """Hide the password so the report is safe to paste into a ticket.""" + return re.sub(r"://([^:/@]+):[^@]*@", r"://\1:***@", url) + + +def scan_dump_text(text: str, table: str = "agent_messages") -> tuple[bool, bool]: + """(defines the table, carries a data section for it) for a plain-SQL dump body.""" + has_ddl = bool(re.search(rf"CREATE TABLE (?:\w+\.)?{re.escape(table)}\b", text)) + has_data = bool( + re.search(rf"^COPY (?:\w+\.)?{re.escape(table)}\b[^\n]*FROM stdin;", text, re.M) + or re.search(rf"^INSERT INTO (?:\w+\.)?{re.escape(table)}\b", text, re.M) + ) + return has_ddl, has_data + + +# --------------------------------------------------------------------------- +# Report model +# --------------------------------------------------------------------------- + + +@dataclass +class CheckResult: + number: int + title: str + status: str + detail: str = "" + remediation: list[str] = field(default_factory=list) + data: dict = field(default_factory=dict) + + def to_dict(self) -> dict: + return { + "number": self.number, + "title": self.title, + "status": self.status, + "detail": self.detail, + "remediation": self.remediation, + "data": self.data, + } + + +class Report: + #: Verdict wording per script. postflight has no "operator judgement" tier: its + #: contract is 0 = verified / 1 = failed, so warnings there exit 0. + VERDICTS = { + "preflight": { + EXIT_OK: "SAFE TO MIGRATE", + EXIT_BLOCKED: "BLOCKED — DO NOT MIGRATE", + EXIT_WARN: "WARNINGS ONLY — operator judgement required", + }, + "postflight": { + EXIT_OK: "VERIFIED", + EXIT_BLOCKED: "VERIFICATION FAILED", + EXIT_WARN: "VERIFIED WITH WARNINGS", + }, + } + + def __init__(self, kind: str, warn_exit_code: int = EXIT_WARN) -> None: + self.kind = kind + self.warn_exit_code = warn_exit_code + self.checks: list[CheckResult] = [] + self.extra: dict = {} + + def add( + self, + title: str, + status: str, + detail: str = "", + remediation: list[str] | None = None, + data: dict | None = None, + ) -> CheckResult: + res = CheckResult( + number=len(self.checks) + 1, + title=title, + status=status, + detail=detail, + remediation=list(remediation or []), + data=dict(data or {}), + ) + self.checks.append(res) + return res + + async def add_guarded(self, title: str, factory): + """Run a check, turning any unexpected exception into a BLOCK item. + + A safety gate that dies with a traceback gives the operator no verdict at all, + which is strictly worse than a loud failure: the temptation is then to migrate + anyway because "the checker is broken". Every check is therefore fenced. + """ + import inspect + + try: + result = factory() + if inspect.isawaitable(result): + result = await result + self.add(*result) + except Exception as exc: # noqa: BLE001 - the exception IS the finding + self.add( + title, + BLOCK, + f"this check could not be completed: {type(exc).__name__}: " + f"{str(exc).strip().splitlines()[0] if str(exc).strip() else exc}", + [ + "Treat an incomplete check as a failed one. Re-run with the full " + "traceback to see why:", + " python scripts/migrate/preflight.py --json # then read the " + "stderr traceback", + ], + {"exception": type(exc).__name__}, + ) + + @property + def statuses(self) -> list[str]: + return [c.status for c in self.checks] + + def exit_code(self) -> int: + if BLOCK in self.statuses: + return EXIT_BLOCKED + if WARN in self.statuses: + return self.warn_exit_code + return EXIT_OK + + def render_text(self) -> str: + lines: list[str] = [] + for c in self.checks: + lines.append(f"{c.number:2d}. [{c.status:5s}] {c.title}") + if c.detail: + for para in c.detail.splitlines(): + lines.append(f" {para}") + for r in c.remediation: + for i, para in enumerate(r.splitlines()): + lines.append((" --> " if i == 0 else " ") + para) + n_block = self.statuses.count(BLOCK) + n_warn = self.statuses.count(WARN) + lines.append("") + table = self.VERDICTS.get(self.kind, self.VERDICTS["preflight"]) + # With warn_exit_code=0 a WARN run still exits 0, so pick the wording off the + # statuses rather than the exit code or a warning would read as a clean pass. + if BLOCK in self.statuses: + verdict = table[EXIT_BLOCKED] + elif WARN in self.statuses: + verdict = table[EXIT_WARN] + else: + verdict = table[EXIT_OK] + lines.append( + f"{verdict} ({len(self.checks)} checks, {n_block} " + f"{'FAIL' if self.kind == 'postflight' else 'BLOCK'}, {n_warn} WARN, " + f"exit {self.exit_code()})" + ) + return "\n".join(lines) + + def to_dict(self) -> dict: + return { + "kind": self.kind, + "exit_code": self.exit_code(), + "verdict": {EXIT_OK: "ok", EXIT_BLOCKED: "blocked", EXIT_WARN: "warn"}[ + worst_status_exit(self.statuses) + ], + "checks": [c.to_dict() for c in self.checks], + **self.extra, + } + + +# --------------------------------------------------------------------------- +# Database access +# --------------------------------------------------------------------------- + + +def resolve_database_url(cli_value: str | None) -> str: + """--database-url, else DATABASE_URL, else the app's configured URL.""" + if cli_value: + return normalize_async_url(cli_value) + env_value = os.environ.get("DATABASE_URL") + if env_value: + return normalize_async_url(env_value) + from src.config import get_settings # imported lazily: keeps the pure logic importable + + return normalize_async_url(get_settings().database_url) + + +async def open_connection(url: str, statement_timeout_ms: int): + """An AUTOCOMMIT connection with bounded waits. + + AUTOCOMMIT matters: a checker that held an open transaction would itself become the + thing that stalls the migration it just blessed. lock_timeout keeps our own catalog + probes from queueing behind somebody else's DDL. + """ + from sqlalchemy import text + from sqlalchemy.ext.asyncio import create_async_engine + + engine = create_async_engine( + url, + isolation_level="AUTOCOMMIT", + pool_pre_ping=False, + connect_args={"server_settings": {"application_name": APPLICATION_NAME}}, + ) + conn = await engine.connect() + await conn.execute(text(f"SET statement_timeout = {int(statement_timeout_ms)}")) + await conn.execute(text("SET lock_timeout = 1000")) + await conn.execute(text("SET idle_in_transaction_session_timeout = 5000")) + return engine, conn + + +async def fetch_all(conn, sql: str, **params) -> list[dict]: + from sqlalchemy import text + + result = await conn.execute(text(sql), params) + return [dict(row) for row in result.mappings()] + + +async def fetch_one_value(conn, sql: str, **params): + from sqlalchemy import text + + result = await conn.execute(text(sql), params) + row = result.first() + return None if row is None else row[0] + + +async def current_revision(conn) -> str | None: + """The stamped revision, or None if the table is missing or empty.""" + exists = await fetch_one_value( + conn, "SELECT to_regclass('public.alembic_version') IS NOT NULL" + ) + if not exists: + return None + return await fetch_one_value(conn, "SELECT version_num FROM alembic_version LIMIT 1") + + +async def table_exists(conn, name: str) -> bool: + return bool(await fetch_one_value(conn, f"SELECT to_regclass('public.{name}') IS NOT NULL")) + + +async def column_exists(conn, table: str, column: str) -> bool: + return bool( + await fetch_one_value( + conn, + "SELECT EXISTS (SELECT 1 FROM information_schema.columns " + "WHERE table_schema='public' AND table_name=:t AND column_name=:c)", + t=table, + c=column, + ) + ) + + +async def existing_object_names(conn) -> dict[str, set[str]]: + """One catalog sweep for every kind of name the chain will try to create.""" + out: dict[str, set[str]] = {"table": set(), "index": set(), "constraint": set(), "type": set()} + for row in await fetch_all( + conn, + # relkind::text, NOT relkind. pg_class.relkind is Postgres' internal "char" type + # and asyncpg decodes it to BYTES (b'r', b'i'), so `row["k"] == "r"` is silently + # always False and this check would fail OPEN — passing every table and index + # collision. Verified: without the cast, a pre-existing + # ix_agent_messages_run_posted (which really does abort migration 0019 with + # DuplicateTableError) was reported as no collision at all. + "SELECT c.relname AS n, c.relkind::text AS k FROM pg_class c " + "JOIN pg_namespace ns ON ns.oid = c.relnamespace WHERE ns.nspname='public'", + ): + if row["k"] == "r": + out["table"].add(row["n"]) + elif row["k"] == "i": + out["index"].add(row["n"]) + for row in await fetch_all( + conn, + "SELECT conname AS n FROM pg_constraint con " + "JOIN pg_namespace ns ON ns.oid = con.connamespace WHERE ns.nspname='public'", + ): + out["constraint"].add(row["n"]) + for row in await fetch_all( + conn, + "SELECT t.typname AS n FROM pg_type t JOIN pg_namespace ns ON ns.oid = t.typnamespace " + "WHERE ns.nspname='public' AND t.typtype='e'", + ): + out["type"].add(row["n"]) + return out + + +async def snapshot_row_counts(conn) -> dict[str, int]: + """Exact counts for every user table. Exact, not reltuples: reltuples is an estimate + that a fresh table reports as -1, which would make the postflight comparison a + coin toss.""" + tables = [r["table_name"] for r in await fetch_all(conn, ROW_COUNT_SQL, schema=SNAPSHOT_SCHEMA)] + counts: dict[str, int] = {} + for t in tables: + counts[t] = int(await fetch_one_value(conn, f'SELECT count(*) FROM public."{t}"')) + return counts + + +# --------------------------------------------------------------------------- +# Backup discovery +# --------------------------------------------------------------------------- + + +def find_backup(path_arg: str | None) -> Path | None: + """A file, the newest matching file in a directory, or the newest in the defaults.""" + candidates: list[Path] = [] + if path_arg: + p = Path(path_arg) + if p.is_file(): + return p + if p.is_dir(): + for pattern in BACKUP_GLOBS: + candidates.extend(p.glob(pattern)) + else: + return None + else: + for d in DEFAULT_BACKUP_DIRS: + base = Path(d) if Path(d).is_absolute() else REPO_ROOT / d + if base.is_dir(): + for pattern in BACKUP_GLOBS: + candidates.extend(base.glob(pattern)) + if not candidates: + return None + return max(candidates, key=lambda p: p.stat().st_mtime) + + +def inspect_backup(path: Path | None, live_rows: int, override: str | None) -> BackupFacts: + facts = BackupFacts(live_agent_messages_rows=live_rows, override_reason=override) + if path is None: + return facts + facts.path = str(path) + if not path.is_file(): + return facts + facts.exists = True + st = path.stat() + facts.size_bytes = st.st_size + facts.age_hours = max(0.0, (time.time() - st.st_mtime) / 3600.0) + try: + with path.open("rb") as fh: + head = fh.read(8) + if head[:2] == b"\x1f\x8b": + facts.fmt = "gzip" + with gzip.open(path, "rt", errors="replace") as gz: + body = gz.read(BACKUP_SCAN_BYTES) + elif head[:5] == b"PGDMP": + facts.fmt = "custom" + return facts + else: + with path.open("rt", errors="replace") as fh: + body = fh.read(BACKUP_SCAN_BYTES) + facts.fmt = "plain" if ("CREATE TABLE" in body or "PostgreSQL database dump" in body) else "unknown" + if facts.fmt == "unknown": + return facts + facts.scanned = True + facts.has_agent_messages_ddl, facts.has_agent_messages_data = scan_dump_text(body) + except (OSError, EOFError, UnicodeError) as exc: + # EOFError, not OSError, is what a TRUNCATED gzip raises ("Compressed file ended + # before the end-of-stream marker was reached") — and a truncated dump is exactly + # the case this check exists for, so crashing on it would be the worst outcome. + # gzip.BadGzipFile is an OSError subclass and is covered by the first arm. + facts.read_error = f"{type(exc).__name__}: {exc}" + return facts + + +# --------------------------------------------------------------------------- +# The checks +# --------------------------------------------------------------------------- + + +async def run_preflight(args) -> Report: + report = Report("preflight") + url = resolve_database_url(args.database_url) + report.extra["database_url"] = redact_url(url) + report.extra["target"] = args.target + report.extra["generated_at"] = time.time() + + engine, conn = await open_connection(url, args.statement_timeout_ms) + try: + # --- 1. current revision ------------------------------------------------- + try: + rev = await current_revision(conn) + except Exception as exc: # noqa: BLE001 + report.add( + "Stamped alembic revision is a supported starting point", + BLOCK, + f"could not read alembic_version: {type(exc).__name__}: {exc}", + ["Check connectivity and permissions; nothing else here can be trusted " + "until this works."], + {}, + ) + return report + status, reason = revision_status(rev, args.target) + remediation: list[str] = [] + if status == BLOCK: + remediation = [ + "Confirm what is actually in the database before doing anything:", + " SELECT * FROM alembic_version;", + "If the database is genuinely empty, create it from scratch instead of " + "migrating: alembic upgrade head.", + "If it is stamped at an unexpected revision, bring it to 0018 or 0019 " + "first and re-run this preflight.", + ] + report.add( + "Stamped alembic revision is a supported starting point", + status, + f"alembic_version = {rev!r}; target {args.target}. {reason}", + remediation, + {"current_revision": rev, "target": args.target}, + ) + report.extra["current_revision"] = rev + + # --- 2. single head, no duplicate revision ids --------------------------- + await report.add_guarded( + "Exactly one alembic head, no duplicate revision ids", + lambda: check_alembic_scripts(rev, args.target), + ) + + # --- 3. is this 0019 the RIGHT 0019? ------------------------------------- + await report.add_guarded( + "The 0019 stamp is the content 0019, not the cohort-branch 0019", + lambda: check_ambiguous_revision(conn, rev), + ) + + # --- 4. THE HARD BLOCKER: duplicate (run, message_ts) ------------------- + await report.add_guarded( + "No duplicate (simulation_run_id, message_ts) in agent_messages", + lambda: check_duplicate_run_ts(conn, rev, args.max_duplicate_groups), + ) + + # --- 5. objects the chain will create that already exist ---------------- + await report.add_guarded( + "Objects the pending revisions create do not already exist", + lambda: check_name_collisions(conn, rev, args.target), + ) + + # --- 6. rows that make a downgrade past 0019 impossible ----------------- + await report.add_guarded( + "Rows that would block a downgrade past 0019 (agent_messages.agent_id IS NULL)", + lambda: check_downgrade_blockers(conn, rev), + ) + + # --- 7. blocking sessions ------------------------------------------------ + await report.add_guarded( + "No sessions that would block (or be blocked by) the ACCESS EXCLUSIVE lock", + lambda: check_blocking_sessions(conn, args.max_xact_age_s), + ) + + # --- 8. migration harness commits what it applies ----------------------- + await report.add_guarded( + "Migration harness commits what it applies (alembic/env.py)", + check_migration_harness, + ) + + # --- 9. sizing / expected lock window ------------------------------------ + rows = 0 + try: + sizing = await check_sizing(conn, rev) + report.add(*sizing) + rows = sizing[4].get("agent_messages_rows", 0) + except Exception as exc: # noqa: BLE001 + report.add( + "Sizing and expected lock window", + BLOCK, + f"this check could not be completed: {type(exc).__name__}: {exc}", + [], + {}, + ) + + # --- 10. index growth headroom ------------------------------------------- + await report.add_guarded( + "Disk headroom for the indexes 0019/0021 add", + lambda: check_index_growth(conn, rev, args.target), + ) + + # --- 11. legacy-row inventory -------------------------------------------- + await report.add_guarded( + "Legacy-row inventory (rows that will have content = '')", + lambda: check_legacy_inventory(conn, rev), + ) + + # --- 12. backup ---------------------------------------------------------- + await report.add_guarded( + "Recent, non-trivial backup exists", lambda: check_backup(args, rows) + ) + + # --- 13. row-count snapshot for postflight ------------------------------- + report.extra["lock_timeout_ms"], report.extra["lock_timeout_source"] = ( + resolve_lock_timeout_ms(dict(os.environ), read_env_py()) + ) + + async def _snapshot_check(): + counts = await snapshot_row_counts(conn) + report.extra["row_counts"] = counts + status_, detail_, rem_ = write_snapshot(args, report, counts, rev) + return ( + "Row-count snapshot written for postflight", + status_, + detail_, + rem_, + {"tables": len(counts), "total_rows": sum(counts.values())}, + ) + + await report.add_guarded("Row-count snapshot written for postflight", _snapshot_check) + finally: + await conn.close() + await engine.dispose() + return report + + +async def check_alembic_scripts(rev: str | None, target: str): + """Mirror scripts/ci.sh's alembic guard, but relate it to the live DB. + + Two migrations sharing a revision id is invisible to git and to pytest, and at + deploy time a targeted ``upgrade `` applies whichever file sorts last while + stamping the database as fully migrated. + """ + versions = sorted((REPO_ROOT / "alembic" / "versions").glob("*.py")) + ids: dict[str, list[str]] = {} + downs: dict[str, str | None] = {} + for f in versions: + src = f.read_text() + m = re.search(r'^revision(?::\s*str)?\s*=\s*["\']([^"\']+)["\']', src, re.M) + d = re.search(r'^down_revision(?::[^=]*)?\s*=\s*(?:["\']([^"\']+)["\']|None)', src, re.M) + if not m: + continue + ids.setdefault(m.group(1), []).append(f.name) + downs[m.group(1)] = d.group(1) if (d and d.group(1)) else None + dupes = {k: v for k, v in ids.items() if len(v) > 1} + parents = {v for v in downs.values() if v} + heads = sorted(set(ids) - parents) + detail = f"{len(versions)} migration files, {len(ids)} revision ids, heads={heads}." + if dupes: + return ( + "Exactly one alembic head, no duplicate revision ids", + BLOCK, + detail + f" DUPLICATE revision ids: {dupes}", + [ + "Renumber the newer migration onto the current head before deploying:", + " grep -h '^revision' alembic/versions/*.py | sort | uniq -d", + "A targeted `alembic upgrade ` on a duplicate-id tree stamps the " + "database as migrated while silently skipping one of the two files.", + ], + {"duplicates": dupes, "heads": heads}, + ) + if len(heads) != 1: + return ( + "Exactly one alembic head, no duplicate revision ids", + BLOCK, + detail + " Expected exactly one head.", + [" python -m alembic heads", "Renumber the newer migration onto the current head."], + {"heads": heads}, + ) + if heads[0] != target: + return ( + "Exactly one alembic head, no duplicate revision ids", + WARN, + detail + f" The single head is {heads[0]}, not the requested target {target}.", + [f"Pass --target {heads[0]}, or migrate to {target} deliberately with " + f"`alembic upgrade {target}`."], + {"heads": heads}, + ) + if rev is not None and rev not in ids: + return ( + "Exactly one alembic head, no duplicate revision ids", + BLOCK, + detail + f" The database is stamped {rev!r}, which no migration file defines.", + ["The stamp does not exist in this tree — you are pointing at a database " + "migrated by a different branch. Do not migrate it from here."], + {"heads": heads, "current_revision": rev}, + ) + return ( + "Exactly one alembic head, no duplicate revision ids", + PASS, + detail, + [], + {"heads": heads}, + ) + + +async def check_ambiguous_revision(conn, rev: str | None): + """A DB stamped 0019 might have been migrated by one of the OTHER 0019s. + + THREE different files in this repository's history declared ``revision = "0019"`` + revising 0018. Enumerated from git, not memory — every blob under + ``alembic/versions/`` in ``git rev-list --all`` was parsed for its declared id: + + * ``0019_agent_message_content.py`` (a7659b4) — the one this chain expects. Adds + ``agent_messages.content`` and six other columns. + * ``0019_add_cohorts.py`` (b00b0e6, branch cohort-agent-isolation) — creates + ``cohorts``/``cohort_memberships``. Verified on a fixture in that exact state: + ``alembic upgrade head`` runs 0020 and 0021 happily (0021's index only touches + columns that exist at 0018) and then dies at 0022 with ``relation "cohorts" + already exists`` — having never applied 0019's content columns, which the app at + HEAD requires. + * ``0019_add_hidden_to_proposals.py`` (4037b79, branch coPI-podcast) — adds + ``hidden`` to ``thread_decisions`` and ``matchmaker_proposals``. Not an ancestor + of main or of this branch, so it is the least likely, but it is on origin and + therefore deployable. + + Naming the wrong culprit sends the operator to the wrong remediation, so probe for + each signature separately rather than assuming the alternative is the cohort one. + """ + title = "The 0019 stamp is the content 0019, not one of the other 0019s" + if rev != "0019": + return (title, PASS, f"not applicable (stamped {rev!r}).", [], {}) + has_content = await column_exists(conn, "agent_messages", "content") + has_cohorts = await table_exists(conn, "cohorts") + has_hidden = await column_exists(conn, "thread_decisions", "hidden") + data = { + "agent_messages.content": has_content, + "cohorts": has_cohorts, + "thread_decisions.hidden": has_hidden, + } + if has_content: + return ( + title, + PASS, + "agent_messages.content is present, so 0019_agent_message_content was the " + "0019 that ran.", + [], + data, + ) + + # Which one actually ran? Say so, and give the matching remediation. + if has_cohorts: + culprit = "0019_add_cohorts (the duplicate id from branch cohort-agent-isolation)" + drops = [" DROP TABLE IF EXISTS cohort_memberships, cohorts CASCADE;"] + extra = [ + "Do not run `alembic upgrade head` as-is: it will apply 0020 and 0021, then " + "abort at 0022 on the already-existing cohorts tables, and the content " + "columns will still be missing.", + ] + elif has_hidden: + culprit = "0019_add_hidden_to_proposals (the duplicate id from branch coPI-podcast)" + drops = [ + " -- Only if you are sure nothing reads them; leaving them is harmless:", + " ALTER TABLE thread_decisions DROP COLUMN IF EXISTS hidden;", + " ALTER TABLE matchmaker_proposals DROP COLUMN IF EXISTS hidden;", + ] + extra = [ + "Those two `hidden` columns are additive and orphaned — nothing in this " + "branch references them. The chain will migrate correctly with them left in " + "place, so prefer leaving them alone over dropping data.", + ] + else: + culprit = "an unrecognised 0019" + drops = [" -- nothing known to drop; investigate before proceeding"] + extra = [ + "Neither the cohorts tables nor thread_decisions.hidden is present, so this " + "matches none of the three 0019s in this repository's history.", + "STOP and inspect the schema by hand. A 0019 stamp with none of the known " + "signatures means this database's history is not one this tooling has seen, " + "and no remediation below is known to be correct for it.", + ] + + return ( + title, + BLOCK, + "stamped 0019 but agent_messages.content does NOT exist: this database was " + f"migrated by {culprit}, so 0019's content columns were never applied.", + [ + *extra, + "Re-stamp to 0018, undo what that other 0019 created, then migrate the " + "whole chain:", + " UPDATE alembic_version SET version_num = '0018';", + *drops, + " -- then: alembic upgrade 0023", + "Take the backup FIRST if you run any of those DROPs; they are destructive.", + ], + data, + ) + + +async def check_duplicate_run_ts(conn, rev: str | None, max_groups: int): + """THE hard blocker. 0019 adds ``uq_agent_messages_run_ts``.""" + title = "No duplicate (simulation_run_id, message_ts) in agent_messages" + if not await table_exists(conn, "agent_messages"): + return (title, WARN, "agent_messages does not exist; nothing to check.", [], {}) + already = await fetch_one_value( + conn, + "SELECT EXISTS (SELECT 1 FROM pg_constraint WHERE conname='uq_agent_messages_run_ts' " + "AND conrelid='agent_messages'::regclass)", + ) + groups = await fetch_all(conn, DUPLICATE_GROUPS_SQL) + n_rows_at_risk = sum(int(g["n"]) - 1 for g in groups) + data = { + "constraint_already_present": bool(already), + "duplicate_group_count": len(groups), + "rows_over_and_above_one_per_group": n_rows_at_risk, + "groups": [ + {"run_id": g["run_id"], "message_ts": g["message_ts"], "n": int(g["n"]), "ids": g["ids"]} + for g in groups[:max_groups] + ], + "groups_truncated": max(0, len(groups) - max_groups), + } + if already: + return ( + title, + PASS, + "uq_agent_messages_run_ts already exists, so the database has been enforcing " + "this since 0019 and no duplicate can be present " + f"(scan confirms {len(groups)} groups).", + [], + data, + ) + if not groups: + return ( + title, + PASS, + "0 duplicate groups. NULL message_ts rows are excluded on purpose: Postgres " + "UNIQUE treats NULLs as distinct, so they cannot violate the new constraint " + "(verified — three NULL-ts rows in one run coexist with the constraint).", + [], + data, + ) + listed = min(len(groups), max_groups) + lines = [ + f"{len(groups)} duplicate group(s), {n_rows_at_risk} row(s) over the one row per " + "group the constraint allows. Postgres names only ONE key per failed index build, " + "so without this list the migration must be run once per group. " + + ( + "All groups:" + if listed == len(groups) + else f"First {listed} of {len(groups)} (raise --max-duplicate-groups for the rest):" + ), + ] + for g in groups[:max_groups]: + lines.append( + f" run {g['run_id']} ts {g['message_ts']} x{int(g['n'])}: {', '.join(g['ids'])}" + ) + if len(groups) > max_groups: + lines.append(f" ... and {len(groups) - max_groups} more (raise --max-duplicate-groups)") + return ( + title, + BLOCK, + "\n".join(lines), + [ + "Inspect before you change anything:", + DUPLICATE_GROUPS_SQL.strip(), + DEDUPE_DELETE_SQL, + DEDUPE_NULL_SQL, + "Then re-run this preflight; it must report 0 groups.", + ], + data, + ) + + +async def check_name_collisions(conn, rev: str | None, target: str): + """Objects the pending revisions CREATE must not already exist. + + None of these creations is guarded. Verified failures on real fixtures: + a hand-made ``ix_agent_messages_run_posted`` gives + ``DuplicateTableError: relation ... already exists``, and an orphaned + ``pi_dm_direction_enum`` type with no pi_dm_messages table gives + ``DuplicateObjectError: type ... already exists`` — 0020 creates the type inline in + ``create_table`` with no ``checkfirst``. + """ + title = "Objects the pending revisions create do not already exist" + planned = planned_objects_between(rev or "0018", target) + if not planned: + return (title, PASS, f"nothing to create for {rev} -> {target}.", [], {}) + existing = await existing_object_names(conn) + collisions: list[PlannedObject] = [] + for obj in planned: + if obj.kind == "column": + if obj.table in existing["table"] and await column_exists(conn, obj.table, obj.name): + collisions.append(obj) + elif obj.name in existing.get(obj.kind, set()): + collisions.append(obj) + data = { + "planned": len(planned), + "collisions": [ + {"revision": o.revision, "kind": o.kind, "name": o.name, "table": o.table} + for o in collisions + ], + } + if not collisions: + return ( + title, + PASS, + f"{len(planned)} object(s) will be created by revisions after {rev}; none exists yet.", + [], + data, + ) + lines = [f"{len(collisions)} name collision(s) — each aborts the chain at its revision:"] + rem = ["Drop the pre-existing objects (they were not created by alembic), or stamp " + "past the revision that creates them if they are genuinely equivalent:"] + for o in collisions: + lines.append(f" {o.revision} would create {o.kind} {o.name}" + + (f" on {o.table}" if o.table else "") + " — already present") + if o.kind == "index": + rem.append(f" DROP INDEX IF EXISTS {o.name};") + elif o.kind == "table": + rem.append(f" DROP TABLE IF EXISTS {o.name} CASCADE; -- destructive, check first") + elif o.kind == "constraint": + rem.append(f" ALTER TABLE {o.table} DROP CONSTRAINT IF EXISTS {o.name};") + elif o.kind == "type": + rem.append(f" DROP TYPE IF EXISTS {o.name};") + elif o.kind == "column": + rem.append(f" -- {o.table}.{o.name} already exists; verify its type matches " + "the migration before stamping past it") + return (title, BLOCK, "\n".join(lines), rem, data) + + +async def check_downgrade_blockers(conn, rev: str | None): + """``agent_messages.agent_id IS NULL`` rows: WARN, and rollback becomes impossible. + + 0019's downgrade does ``ALTER COLUMN agent_id SET NOT NULL``. Any PI/human row + (agent_id NULL by design from 0019 on) makes that statement fail, so once such a row + exists a downgrade past 0019 cannot run at all. At 0018 the column is still NOT NULL, + so the count is necessarily 0 and this check is informational. + """ + title = "Rows that would block a downgrade past 0019 (agent_messages.agent_id IS NULL)" + if not await table_exists(conn, "agent_messages"): + return (title, PASS, "agent_messages does not exist.", [], {}) + nullable = await fetch_one_value( + conn, + "SELECT NOT attnotnull FROM pg_attribute WHERE attrelid='agent_messages'::regclass " + "AND attname='agent_id'", + ) + if not nullable: + return ( + title, + PASS, + "agent_id is still NOT NULL at this revision, so no such row can exist. " + "Note that once 0019 has run, every PI/human message will have agent_id NULL " + "and this becomes a one-way door.", + [], + {"agent_id_nullable": False, "null_agent_id_rows": 0}, + ) + n = int(await fetch_one_value(conn, "SELECT count(*) FROM agent_messages WHERE agent_id IS NULL")) + data = {"agent_id_nullable": True, "null_agent_id_rows": n} + if n == 0: + return (title, PASS, "0 rows with agent_id IS NULL.", [], data) + return ( + title, + WARN, + f"{n:,} row(s) have agent_id IS NULL (PI/human messages). 0019's downgrade runs " + "ALTER COLUMN agent_id SET NOT NULL, which these rows make fail: ROLLBACK PAST " + "0019 IS IMPOSSIBLE while they exist. This is a WARN, not a BLOCK — it does not " + "affect the upgrade.", + [ + "Nothing to fix before the upgrade. Know that your only rollback is a restore " + "from the backup, not `alembic downgrade`.", + "To see them:", + " SELECT id, simulation_run_id, sender_name, channel_name, message_ts\n" + " FROM agent_messages WHERE agent_id IS NULL ORDER BY created_at;", + ], + data, + ) + + +async def check_blocking_sessions(conn, max_xact_age_s: float = DEFAULT_MAX_TOLERABLE_XACT_AGE_S): + """Report every session that could hold a conflicting lock.""" + title = "No sessions that would block (or be blocked by) the ACCESS EXCLUSIVE lock" + sessions = await fetch_all(conn, BLOCKING_SESSIONS_SQL, app_name=APPLICATION_NAME) + status, detail = blocking_sessions_status(sessions, max_xact_age_s) + lines = [detail] + for s in sessions: + lines.append( + f" pid {s['pid']} state={s['state']} app={s['application_name']!r} " + f"xact_age={s['xact_age_s']:.1f}s query_age={s['query_age_s']:.1f}s " + f"holds_lock_on_agent_messages={s['holds_agent_messages_lock']}" + ) + lines.append(f" query: {str(s['query'])[:200]}") + rem: list[str] = [] + if status != PASS: + rem = [ + "Stop the writers first — the agent simulation is the main one, and it must be " + "stopped GRACEFULLY or the in-flight turn's messages are lost:", + " docker stop -t 30 agent-run", + " docker compose stop app worker grantbot", + "Then re-check, and only terminate what is left if you know what it is:", + " SELECT pid, state, now()-xact_start AS age, query FROM pg_stat_activity\n" + " WHERE datname = current_database() AND xact_start IS NOT NULL;", + " SELECT pg_terminate_backend();", + ] + return ( + title, + status, + "\n".join(lines), + rem, + {"sessions": sessions, "session_count": len(sessions)}, + ) + + +def read_env_py() -> str | None: + p = REPO_ROOT / "alembic" / "env.py" + try: + return p.read_text() + except OSError: + return None + + +def check_migration_harness(): + """Does ``alembic upgrade`` actually commit what it applies? + + See ``harness_findings``. This is the only check here that can fail while every row + of data is perfect, and it is the most dangerous failure of the lot because the + symptom is a *successful-looking* migration. + """ + title = "Migration harness commits what it applies (alembic/env.py)" + src = read_env_py() + if src is None: + return ( + title, + WARN, + "could not read alembic/env.py, so the harness could not be checked.", + ["Run preflight from a checkout of the tree you are migrating with."], + {}, + ) + findings = harness_findings(src) + lock_ms, lock_src = resolve_lock_timeout_ms(dict(os.environ), src) + data = {"findings": findings, "lock_timeout_ms": lock_ms, "lock_timeout_source": lock_src} + if not findings: + return ( + title, + PASS, + f"do_run_migrations issues no SQL before context.begin_transaction(). " + f"Effective lock_timeout: {lock_ms} ms ({lock_src}).", + [], + data, + ) + return ( + title, + BLOCK, + "\n".join( + [ + "`alembic upgrade` on this tree will log a full successful chain, exit 0, " + "and COMMIT NOTHING:", + *(f" {f}" for f in findings), + "Mechanism: the early statement autobegins a transaction; " + "MigrationContext.__init__ then sets _in_external_transaction=True, " + "begin_transaction() degrades to nullcontext(), alembic leaves the commit " + "to the caller, and run_async_migrations()'s connect() block rolls back on " + "exit.", + f"Effective lock_timeout: {lock_ms} ms ({lock_src}).", + ] + ), + [ + "Set the timeout on the ENGINE instead of on the connection, so no statement " + "runs before alembic demarcates its transaction — e.g. pass it in the DSN or " + "via connect_args, or issue it inside do_run_migrations AFTER " + "context.begin_transaction() has been entered.", + "Verify the fix the only way that counts — on a throwaway database, confirm " + "the version actually moved:", + " DATABASE_URL= python -m alembic upgrade head", + " psql -c 'SELECT version_num FROM alembic_version'", + "Until it is fixed you can neutralise it with ALEMBIC_LOCK_TIMEOUT_MS=0, which " + "skips the offending statement (but then the migration waits for locks forever " + "— see check on blocking sessions).", + "Run postflight after every migration regardless: it is what catches this.", + ], + data, + ) + + +async def check_sizing(conn, rev: str | None = None): + """agent_messages row count, size, and the estimated lock window. + + The estimate is a function of the 0019 index build, so it only applies when 0019 is + still pending. Starting from 0020/0021 that cost is already paid and the remaining + chain (0022's three empty tables, 0023's three columns on a small table) does not + scale with agent_messages at all — quoting the row-scaled number there would tell an + operator to book an outage they do not need. + """ + title = "Sizing and expected lock window" + if not await table_exists(conn, "agent_messages"): + return (title, WARN, "agent_messages does not exist.", [], {"agent_messages_rows": 0}) + rows = int(await fetch_one_value(conn, "SELECT count(*) FROM agent_messages")) + if rev in POST_0019_STARTS: + heap = int(await fetch_one_value(conn, "SELECT pg_relation_size('agent_messages')")) + return ( + title, + PASS, + f"agent_messages: {rows:,} rows, heap {heap / 1e6:.1f} MB — but 0019 has " + f"already run at {rev}, so its ACCESS EXCLUSIVE index build is behind you. " + f"What remains is 0022 (three empty tables) and 0023 (three columns on " + f"researcher_profiles); neither scales with agent_messages. Measured at ~2s " + f"at every size tested.", + [], + { + "agent_messages_rows": rows, + "agent_messages_heap_bytes": heap, + "estimated_lock_window_ms_low": 0, + "estimated_lock_window_ms_high": 2000, + "index_build_already_done": True, + }, + ) + heap = int(await fetch_one_value(conn, "SELECT pg_relation_size('agent_messages')")) + total = int(await fetch_one_value(conn, "SELECT pg_total_relation_size('agent_messages')")) + dbsize = int(await fetch_one_value(conn, "SELECT pg_database_size(current_database())")) + lo, hi = estimate_lock_window_ms(rows) + status, note = sizing_status(rows, hi) + data = { + "agent_messages_rows": rows, + "agent_messages_heap_bytes": heap, + "agent_messages_total_relation_bytes": total, + "database_bytes": dbsize, + "estimated_lock_window_ms_low": lo, + "estimated_lock_window_ms_high": hi, + } + detail = ( + f"agent_messages: {rows:,} rows, heap {heap / 1e6:.1f} MB, total relation " + f"{total / 1e6:.1f} MB; database {dbsize / 1e6:.1f} MB.\n" + f"Estimated ACCESS EXCLUSIVE window {lo / 1000:.1f}s (idle server, warm cache) to " + f"{hi / 1000:.1f}s (busy server). {note}\n" + f"The whole 0019..0023 chain runs in ONE transaction, so the lock is held for the " + f"entire chain, not just the index build. Calibrated at 10k/100k/1M rows: " + f"112 ms / 747 ms / 7,902 ms." + ) + rem = [] + if status != PASS: + rem = [ + "Announce the window and stop the writers for its duration:", + " docker stop -t 30 agent-run && docker compose stop app worker grantbot", + "There is no CONCURRENTLY option available here: alembic runs the whole chain " + "in one transaction and CREATE INDEX CONCURRENTLY cannot run inside one.", + ] + return (title, status, detail, rem, data) + + +async def check_index_growth(conn, rev: str | None, target: str): + """The four new indexes cost about +80% of the current relation size.""" + title = "Disk headroom for the indexes 0019/0021 add" + planned = planned_objects_between(rev or "0018", target) + new_idx = [o for o in planned if o.kind == "index" and o.table == "agent_messages"] + if not new_idx or not await table_exists(conn, "agent_messages"): + return (title, PASS, "no new agent_messages indexes for this transition.", [], {}) + total = int(await fetch_one_value(conn, "SELECT pg_total_relation_size('agent_messages')")) + need = int(total * INDEX_GROWTH_FRACTION) + data = { + "new_indexes": [o.name for o in new_idx], + "current_total_relation_bytes": total, + "estimated_additional_bytes": need, + } + detail = ( + f"{len(new_idx)} new index(es) on agent_messages: {', '.join(o.name for o in new_idx)}. " + f"Measured growth at three scales was +79%, +79%, +80% of the pre-migration total " + f"relation size, so budget about {need / 1e6:.0f} MB of new index data plus sort " + f"space for the build." + ) + if need > INDEX_GROWTH_WARN_BYTES: + return ( + title, + WARN, + detail + " That is over 1 GiB; preflight cannot see the filesystem from inside " + "Postgres, so confirm free space by hand.", + [" docker compose exec postgres df -h /var/lib/postgresql/data"], + data, + ) + return (title, PASS, detail, [], data) + + +async def check_legacy_inventory(conn, rev: str | None): + """Rows that end up with ``content = ''``, split by whether Slack still has them.""" + title = "Legacy-row inventory (rows that will have content = '')" + if not await table_exists(conn, "agent_messages"): + return (title, PASS, "agent_messages does not exist.", [], {}) + has_content = await column_exists(conn, "agent_messages", "content") + if has_content: + where = "content = ''" + else: + # Pre-0019: the column does not exist yet, so EVERY row gets the server_default ''. + where = "TRUE" + recoverable = int( + await fetch_one_value( + conn, + f"SELECT count(*) FROM agent_messages WHERE {where} AND channel_id NOT LIKE 'local:%'", + ) + ) + unrecoverable = int( + await fetch_one_value( + conn, + f"SELECT count(*) FROM agent_messages WHERE {where} AND channel_id LIKE 'local:%'", + ) + ) + status, note = legacy_inventory_status(recoverable, unrecoverable) + data = { + "content_column_present": bool(has_content), + "empty_content_slack_recoverable": recoverable, + "empty_content_unrecoverable": unrecoverable, + } + detail = note + ( + "" + if has_content + else " (agent_messages.content does not exist yet, so every existing row will take " + "the server_default '' — the message bodies were never in this database.)" + ) + rem: list[str] = [] + if recoverable: + rem.append( + "The Slack-side rows can be recovered, with Slack tokens available, by:\n" + " docker compose exec app python scripts/backfill_slack_history_to_db.py\n" + "It upserts on (simulation_run_id, message_ts) and is safe to re-run." + ) + if unrecoverable: + rem.append( + f"The {unrecoverable:,} 'local:' rows were never mirrored anywhere. Their bodies " + "do not exist; they will read as empty messages forever. Do not let anyone " + "'fix' this by inventing content." + ) + return (title, status, detail, rem, data) + + +def check_backup(args, live_rows: int): + """A recent, data-bearing dump must exist: rollback past 0019 is destructive.""" + title = "Recent, non-trivial backup exists" + path = find_backup(args.backup_path) + facts = inspect_backup(path, live_rows, args.backup_verified_elsewhere) + status, notes = evaluate_backup(facts, args.backup_max_age_hours, args.backup_min_bytes) + detail = notes[0] if notes else "" + rem = list(notes[1:]) + return ( + title, + status, + detail, + rem, + { + "path": facts.path, + "exists": facts.exists, + "size_bytes": facts.size_bytes, + "age_hours": facts.age_hours, + "format": facts.fmt, + "has_agent_messages_data": facts.has_agent_messages_data, + "override_reason": facts.override_reason, + }, + ) + + +def write_snapshot(args, report: Report, counts: dict[str, int], rev: str | None): + """Hand off to postflight. The snapshot is the only thing postflight cannot re-derive.""" + payload = { + "kind": "preflight-snapshot", + "generated_at": time.time(), + "database_url": redact_url(report.extra.get("database_url", "")), + "current_revision": rev, + "target": args.target, + "row_counts": counts, + } + detail = ( + f"{len(counts)} tables, {sum(counts.values()):,} rows total (exact counts, not " + "reltuples)." + ) + if not args.snapshot: + return ( + WARN, + detail + " No --snapshot path given, so postflight cannot compare row counts.", + [ + "Re-run with --snapshot to enable the postflight row-count comparison:", + " python scripts/migrate/preflight.py --snapshot " + "/app/logs/migration_snapshot.json", + "(the counts are also in the --json output under row_counts)", + ], + ) + try: + p = Path(args.snapshot) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps(payload, indent=2, sort_keys=True)) + except OSError as exc: + return (BLOCK, f"could not write snapshot to {args.snapshot}: {exc}", [ + "postflight cannot verify row counts without it; pick a writable path." + ]) + return (PASS, detail + f" Written to {args.snapshot}.", []) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def add_common_arguments(ap: argparse.ArgumentParser) -> None: + ap.add_argument( + "--database-url", + default=None, + help="Target database. Default: $DATABASE_URL, else the app's configured URL.", + ) + ap.add_argument("--target", default=DEFAULT_TARGET, help=f"Target revision (default {DEFAULT_TARGET})") + ap.add_argument("--json", action="store_true", help="Also print a machine-readable JSON report") + ap.add_argument( + "--json-out", default=None, help="Write the JSON report to this path as well as stdout" + ) + ap.add_argument( + "--statement-timeout-ms", + type=int, + default=60_000, + help="Bound on each of this script's own queries (default 60000)", + ) + + +def build_parser() -> argparse.ArgumentParser: + ap = argparse.ArgumentParser( + prog="preflight", + description="Pre-migration safety gate. Exit 0 = safe, 1 = BLOCKED, 2 = warnings only.", + ) + add_common_arguments(ap) + ap.add_argument("--snapshot", default=None, help="Write the row-count snapshot postflight reads") + ap.add_argument( + "--backup-path", + default=None, + help="Backup file, or a directory to take the newest dump from. " + f"Default: newest match under {', '.join(DEFAULT_BACKUP_DIRS)}", + ) + ap.add_argument( + "--backup-max-age-hours", + type=float, + default=DEFAULT_BACKUP_MAX_AGE_HOURS, + help=f"Reject a backup older than this (default {DEFAULT_BACKUP_MAX_AGE_HOURS:.0f})", + ) + ap.add_argument( + "--backup-min-bytes", + type=int, + default=DEFAULT_BACKUP_MIN_BYTES, + help=f"Reject a backup smaller than this (default {DEFAULT_BACKUP_MIN_BYTES})", + ) + ap.add_argument( + "--backup-verified-elsewhere", + default=None, + metavar="REASON", + help="Downgrade the backup check to WARN, recording REASON in the report. " + "For managed snapshots preflight cannot see. Not a way to skip having a backup.", + ) + ap.add_argument( + "--max-duplicate-groups", + type=int, + default=200, + help="Cap on duplicate groups listed individually (default 200)", + ) + ap.add_argument( + "--max-xact-age-s", + type=float, + default=DEFAULT_MAX_TOLERABLE_XACT_AGE_S, + help="An `active` session with a transaction older than this BLOCKs " + f"(default {DEFAULT_MAX_TOLERABLE_XACT_AGE_S:.0f}). Idle-in-transaction always BLOCKs.", + ) + return ap + + +def emit(report: Report, args) -> int: + print(f"=== {report.kind}: {report.extra.get('database_url')} " + f"(target {report.extra.get('target')}) ===") + print(report.render_text()) + payload = report.to_dict() + if args.json: + print("--- JSON ---") + print(json.dumps(payload, indent=2, sort_keys=True, default=str)) + if args.json_out: + Path(args.json_out).parent.mkdir(parents=True, exist_ok=True) + Path(args.json_out).write_text(json.dumps(payload, indent=2, sort_keys=True, default=str)) + return report.exit_code() + + +def main(argv: list[str] | None = None) -> int: + import asyncio + + args = build_parser().parse_args(argv) + report = asyncio.run(run_preflight(args)) + return emit(report, args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/migrate/remediate_duplicates.py b/scripts/migrate/remediate_duplicates.py new file mode 100644 index 0000000..e87504f --- /dev/null +++ b/scripts/migrate/remediate_duplicates.py @@ -0,0 +1,1287 @@ +"""Clear duplicate ``(simulation_run_id, message_ts)`` rows so 0019 can apply. + +WHY THIS EXISTS +--------------- +Migration 0019 ends with:: + + op.create_unique_constraint( + "uq_agent_messages_run_ts", "agent_messages", ["simulation_run_id", "message_ts"] + ) + +On a production database at 0018 that holds two or more ``agent_messages`` rows +sharing a non-NULL ``(simulation_run_id, message_ts)``, that statement aborts:: + + UniqueViolationError: could not create unique index "uq_agent_messages_run_ts" + DETAIL: Key (simulation_run_id, message_ts)=(…) is duplicated. + +Three measured facts about that failure shape this tool (all reproduced on a +throwaway Postgres 15 at 0018, see the report accompanying this change): + +1. ``alembic/env.py`` deliberately runs the whole chain in ONE transaction, so a + failure rolls the database all the way back to 0018 — nothing is half-applied, + but nothing is gained either. +2. The error names exactly ONE duplicate group per attempt. With 9 duplicate + groups it took 10 ``ALTER TABLE`` attempts to walk them all. +3. That ``ALTER TABLE`` requests **AccessExclusiveLock** on ``agent_messages`` + (verified in ``pg_locks``: it blocks behind a single open ``BEGIN; SELECT``). + A pending ACCESS EXCLUSIVE request also queues ahead of new readers, so every + attempt is a stall on the hot table. Iterating on migration failures therefore + costs one ACCESS EXCLUSIVE lock cycle per duplicate group. Remediating up + front costs one. + +Rows with ``message_ts IS NULL`` are NOT affected: the index Postgres builds is +NULLS DISTINCT (``pg_index.indnullsnotdistinct = false``), so any number of NULL +``message_ts`` rows coexist. Verified with 65 such rows present while the +constraint was created successfully. This tool never looks at them. + +DELETE vs RENUMBER +------------------ +Deleting a row destroys conversation history; the DB is the durable store from +0019 on, so a dropped message is unrecoverable. Renumbering a ``message_ts`` +that is a real Slack timestamp destroys the only record of that timestamp while +the ``slack_ts`` column does not yet exist (0018), which is the same +timestamp-fabrication mistake ``scripts/backfill_slack_ts.py`` was written to +undo. So neither action is safe in general, and the tool decides per group: + +* A group whose rows are **payload-identical** (equal on every column except + ``id`` and ``created_at``) is one message logged twice. The extra rows carry + nothing the survivor does not, so DELETING them loses no information and is + the semantically correct repair — but it is destructive, so it happens only + under ``--strategy keep-earliest`` / ``keep-latest``. +* A **divergent** group (the rows differ in content, sender, channel, length…) + is never resolved by deletion, under any strategy. Each row may carry + something unique, so the rows are preserved and the ones whose ``message_ts`` + may safely change are RENUMBERED. +* "May safely change" is decided per row by ``classify_origin`` below. A + ``message_ts`` that is (or may be) a Slack-issued timestamp and is recorded + nowhere else must not change. +* A group where **two or more** rows look Slack-born and the rows are not + payload-identical cannot be repaired without guessing which row's timestamp is + the wrong one. The tool REFUSES it and tells the operator exactly what to look + at. It never guesses. + +Renumbered ids are minted in a writer slot no live minter owns +(``REMEDIATION_WRITER_SLOT``), in the same microsecond neighbourhood as the id +they replace, and checked against every ``message_ts``/``thread_ts``/ +``thread_decisions.thread_id`` already present in that run. See +``mint_replacement_ts``. + +USAGE +----- +Dry run (default — writes nothing, in a READ ONLY transaction):: + + docker compose exec -e PYTHONPATH=/app app \\ + python scripts/migrate/remediate_duplicates.py + +Apply:: + + docker compose exec -e PYTHONPATH=/app app \\ + python scripts/migrate/remediate_duplicates.py --apply --strategy keep-earliest + +``PYTHONPATH=/app`` is REQUIRED and is not decoration. ``python /x.py`` puts +the *script's* directory on ``sys.path[0]``, not the repo root, so ``import src`` +resolves to the copy baked into the image at build time +(``/usr/local/lib/python3.11/site-packages/src``) rather than the mounted +``/app/src``. Verified inside ``copiscience-app-1``. The header this tool prints +names the ``src/agent/ids.py`` it actually loaded, so you can see which copy you +got, and it hard-fails if that copy's writer-slot scheme is not the one it +expects. + +EXIT CODES +---------- +0 no duplicates, or ``--apply`` finished and none remain (re-verified in the + same transaction as the writes). +1 duplicates remain, or would remain under the chosen strategy. +2 duplicates found in a dry run and the plan resolves all of them. A runbook + can branch on this: 2 means "there is work to do and it is safe to do". +3 operational failure (no DSN, database unreachable, ``agent_messages`` + missing, lock timeout, id scheme unrecognised). Deliberately not 1 or 2 so a + runbook cannot mistake a broken run for a verdict about the data. +64 usage error (``EX_USAGE``). argparse's own default is 2, which would be + indistinguishable from "duplicates found"; it is remapped. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +import time +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, NoReturn + +# --------------------------------------------------------------------------- # +# The id scheme. Imported, not copied, so it cannot drift from the minters -- +# and then validated, because the import may have come from a stale baked copy +# of src/ (see the PYTHONPATH note in the module docstring). +# --------------------------------------------------------------------------- # + +#: Writer slot used for renumbered ids. ``src/agent/ids.py`` gives every minter a +#: residue class of the microsecond field so two processes can never mint the +#: same id; 0-3 are claimed by the engine, the web app, GrantBot and the engine's +#: module-default minter. 99 is claimed here, at the far end of the range, so a +#: remediated id cannot collide with anything a running system mints -- and so a +#: renumbered id is recognisable as remediated at a glance. +REMEDIATION_WRITER_SLOT = 99 + +#: What this tool assumes ``src/agent/ids.py`` uses. If the loaded module says +#: something else, the collision argument above is void and we stop. +EXPECTED_SLOT_MODULUS = 100 + +#: A ts-shaped id: ".". Both Slack +#: timestamps and locally minted ids take this form. Anything else was never +#: issued by Slack. +#: +#: ``\Z``, not ``$``: ``$`` also matches immediately BEFORE a trailing newline, so +#: ``"1755000005.000001\n"`` matched and was then treated as a well-formed id -- +#: which would have had the tool renumber a neighbouring row rather than the +#: malformed one, and compare a normalised id against an un-normalised column. +#: ``[0-9]`` rather than ``\d`` for the same reason: ``\d`` matches Unicode digits +#: that ``int()`` accepts but Slack never issues. +TS_SHAPE = re.compile(r"\A[0-9]{1,19}\.[0-9]{6}\Z") + +#: Channel ids for channels that exist only in the DB. ``SimulationEngine`` +#: writes ``f"local:{channel}"`` when there is no Slack channel to mirror into, +#: so a row with this prefix was never posted to Slack and its ``message_ts`` +#: cannot be a Slack timestamp. Same predicate ``scripts/backfill_slack_ts.py`` +#: uses to skip rows it must not ask Slack about. +LOCAL_CHANNEL_PREFIX = "local:" + +#: Columns that do NOT count as payload when deciding whether two rows are the +#: same message logged twice. ``id`` must differ (it is the primary key) and +#: ``created_at`` is the DB's own bookkeeping clock, not the message. +NON_PAYLOAD_COLUMNS = frozenset({"id", "created_at"}) + +TABLE = "agent_messages" +CONSTRAINT = "uq_agent_messages_run_ts" + +EXIT_CLEAN = 0 +EXIT_REMAIN = 1 +EXIT_FOUND_DRY_RUN = 2 +EXIT_OPERATIONAL = 3 +EXIT_USAGE = 64 + + +def load_id_scheme() -> tuple[int, dict[int, str], str]: + """Return ``(modulus, {slot: writer name}, module path)`` from src.agent.ids. + + Raises ``SchemeError`` if the module cannot be imported or if its scheme is + not the one ``REMEDIATION_WRITER_SLOT`` was chosen against. Failing loudly + here is the point: a silently stale ``src`` could hand back a different + modulus, and renumbered ids would then land in a slot a live minter owns. + """ + try: + from src.agent import ids as ids_mod + except ImportError as exc: # pragma: no cover - exercised by hand, not in CI + raise SchemeError( + f"cannot import src.agent.ids ({exc}). Run this from the repo root, or " + "inside the container with PYTHONPATH=/app:\n" + " docker compose exec -e PYTHONPATH=/app app " + "python scripts/migrate/remediate_duplicates.py" + ) from exc + + modulus = getattr(ids_mod, "WRITER_SLOT_MODULUS", None) + if modulus != EXPECTED_SLOT_MODULUS: + raise SchemeError( + f"src.agent.ids.WRITER_SLOT_MODULUS is {modulus!r}, expected " + f"{EXPECTED_SLOT_MODULUS}. REMEDIATION_WRITER_SLOT=" + f"{REMEDIATION_WRITER_SLOT} was chosen against the latter; with a " + "different modulus a renumbered id could land in a live minter's " + f"slot. Loaded from {getattr(ids_mod, '__file__', '?')}." + ) + + slots: dict[int, str] = {} + for name in dir(ids_mod): + if not name.startswith("WRITER_") or name == "WRITER_SLOT_MODULUS": + continue + value = getattr(ids_mod, name) + if isinstance(value, int) and not isinstance(value, bool): + slots[value] = name + if not slots: + raise SchemeError( + "src.agent.ids declares no WRITER_* slots; refusing to guess which " + "residue classes are in use." + ) + if REMEDIATION_WRITER_SLOT in slots: + raise SchemeError( + f"writer slot {REMEDIATION_WRITER_SLOT} is now claimed by " + f"{slots[REMEDIATION_WRITER_SLOT]}. Pick a free slot for " + "REMEDIATION_WRITER_SLOT before running this." + ) + return modulus, slots, getattr(ids_mod, "__file__", "?") + + +class SchemeError(RuntimeError): + """The loaded id scheme is not the one renumbering was designed against.""" + + +# --------------------------------------------------------------------------- # +# Pure logic. No DB, no I/O -- this is what tests/unit/test_remediate_duplicates.py +# exercises. +# --------------------------------------------------------------------------- # + +# Origin verdicts, in descending order of confidence. +ORIGIN_LOCAL_CONFIRMED = "local_confirmed" +ORIGIN_SLACK_CONFIRMED = "slack_confirmed" +ORIGIN_LOCAL_PRESUMED = "local_presumed" +ORIGIN_SLACK_PRESUMED = "slack_presumed" + +RENUMBER_SAFE = "safe" +RENUMBER_UNSAFE = "unsafe" + +KIND_REDUNDANT = "redundant" +KIND_DIVERGENT = "divergent" + +RESOLUTION_DELETE = "delete" +RESOLUTION_RENUMBER = "renumber" +RESOLUTION_NEEDS_DELETE_STRATEGY = "needs_delete_strategy" +RESOLUTION_NEEDS_HUMAN = "needs_human" + +UNRESOLVED = frozenset({RESOLUTION_NEEDS_DELETE_STRATEGY, RESOLUTION_NEEDS_HUMAN}) + +STRATEGY_KEEP_EARLIEST = "keep-earliest" +STRATEGY_KEEP_LATEST = "keep-latest" +STRATEGY_RENUMBER = "renumber" +STRATEGIES = (STRATEGY_KEEP_EARLIEST, STRATEGY_KEEP_LATEST, STRATEGY_RENUMBER) + +ACTION_KEEP = "keep" +ACTION_RENUMBER = "renumber" +ACTION_DELETE = "delete" + + +def parse_ts_us(ts: str | None) -> int | None: + """Return integer microseconds-since-epoch for a ts-shaped id, else None. + + Only the exact ``.<6 digits>`` form is accepted. ``"1755000000.2"`` + is rejected rather than guessed at: 2 microseconds and 200000 microseconds + are both plausible readings and picking one silently would move a message by + a fifth of a second. + """ + if not ts or not TS_SHAPE.match(ts): + return None + seconds, _, micros = ts.partition(".") + return int(seconds) * 1_000_000 + int(micros) + + +def format_us(us: int) -> str: + """Format integer microseconds as a ts-shaped id. + + Byte-identical to ``src.agent.ids._fmt``; the unit tests pin that agreement + so the two cannot drift. + """ + return f"{us // 1_000_000}.{us % 1_000_000:06d}" + + +@dataclass(frozen=True) +class Origin: + verdict: str + evidence: str + + @property + def is_slack(self) -> bool: + return self.verdict in (ORIGIN_SLACK_CONFIRMED, ORIGIN_SLACK_PRESUMED) + + +@dataclass +class MessageRow: + """One ``agent_messages`` row, revision-agnostic. + + ``columns`` holds every column the database actually has, so the same code + works at 0018 (no ``content``/``slack_ts``) and at 0019 (both present). + """ + + row_id: str + run_id: str + message_ts: str + created_at: datetime | None + columns: dict[str, Any] + + # Filled in by planning. + origin: Origin | None = None + renumber_verdict: str = RENUMBER_UNSAFE + renumber_reason: str = "" + action: str = ACTION_KEEP + new_message_ts: str | None = None + + @property + def channel_id(self) -> str: + return str(self.columns.get("channel_id") or "") + + @property + def slack_ts(self) -> str | None: + value = self.columns.get("slack_ts") + return None if value is None else str(value) + + def payload(self) -> tuple: + """The row's identity for "is this the same message twice?". + + Values are compared by ``repr``, not by ``==``. That is deliberately + STRICTER than equality (it will not call ``Decimal("1")`` equal to ``1``) + and it is total: every column type reprs, including a JSON column that + would be unhashable as a set member. Erring strict means a row is judged + "redundant" -- and therefore deletable -- only when it really is + indistinguishable. + """ + return tuple( + (name, repr(value)) + for name, value in sorted(self.columns.items()) + if name not in NON_PAYLOAD_COLUMNS + ) + + def sort_key(self) -> tuple: + """Deterministic ordering: DB insert clock, then primary key.""" + # created_at is NOT NULL in the schema, but a NULL would otherwise make + # the comparison explode rather than just sort first. + return (self.created_at is not None, self.created_at, self.row_id) + + +def classify_origin(row: MessageRow, *, has_slack_columns: bool, writer_slots: dict[int, str], + modulus: int) -> Origin: + """Decide whether this row's ``message_ts`` is a Slack timestamp. + + Ordered strongest evidence first. The two "confirmed" verdicts are facts + about the row; the two "presumed" ones are judgement calls, and the tool says + which it used for every row it reports. + """ + ts = row.message_ts + + # 1. The canonical id and the Slack ts are already recorded separately and + # they differ, so message_ts is a local id by construction (0019+). + if has_slack_columns and row.slack_ts is not None and row.slack_ts != ts: + return Origin( + ORIGIN_LOCAL_CONFIRMED, + f"slack_ts={row.slack_ts} differs from message_ts, so the canonical id " + "is already decoupled from the Slack timestamp", + ) + + # 2. The message never went to Slack, so no Slack ts exists to protect. + if row.channel_id.startswith(LOCAL_CHANNEL_PREFIX): + return Origin( + ORIGIN_LOCAL_CONFIRMED, + f"channel_id {row.channel_id!r} is DB-only, so this message was never " + "posted to Slack", + ) + + # 3. Slack never issues an id of this shape. + us = parse_ts_us(ts) + if us is None: + return Origin( + ORIGIN_LOCAL_CONFIRMED, + f"message_ts {ts!r} is not ts-shaped (.<6 digits>); Slack " + "never issues this form", + ) + + # 4. Confirmed Slack ts. Renumbering message_ts is still safe here because + # slack_ts keeps the Slack timestamp -- see renumber_verdict(). + if has_slack_columns and row.slack_ts == ts: + return Origin(ORIGIN_SLACK_CONFIRMED, "slack_ts == message_ts") + + # 5. Writer-slot residue. This is the ONLY signal available at 0018 for a + # locally minted id that also carries a real Slack channel id -- a PI + # message written through the web inbox, or an agent post whose Slack + # mirror failed (both called out in scripts/backfill_slack_ts.py). It is a + # judgement call: a random Slack ts lands in one of the four claimed slots + # about 4% of the time. + residue = us % modulus + if residue in writer_slots: + return Origin( + ORIGIN_LOCAL_PRESUMED, + f"microsecond residue {residue:02d} is writer slot " + f"{writer_slots[residue]} (src/agent/ids.py), so this looks locally " + "minted despite the Slack channel id", + ) + + # 6. Nothing says local, and it is a ts-shaped id in a real Slack channel. + return Origin( + ORIGIN_SLACK_PRESUMED, + f"ts-shaped id in Slack channel {row.channel_id}, microsecond residue " + f"{residue:02d} matches no writer slot", + ) + + +def renumber_verdict(row: MessageRow, origin: Origin, *, has_slack_columns: bool) -> tuple[str, str]: + """May this row's ``message_ts`` be changed without losing information?""" + if origin.verdict == ORIGIN_LOCAL_CONFIRMED: + return RENUMBER_SAFE, "no Slack-issued timestamp to lose" + if origin.verdict == ORIGIN_LOCAL_PRESUMED: + return RENUMBER_SAFE, ( + "presumed locally minted from its writer slot; if that presumption is " + "wrong the cost is a lost Slack-mirror mapping, recoverable with " + "scripts/backfill_slack_ts.py" + ) + if origin.verdict == ORIGIN_SLACK_CONFIRMED: + # slack_ts is a separate column from 0019 on, so the Slack timestamp + # survives a change to message_ts. This is exactly what 0018 cannot do. + return RENUMBER_SAFE, "slack_ts keeps the Slack timestamp; only the canonical id changes" + where = "slack_ts" if has_slack_columns else "the 0018 schema has no slack_ts column, so" + return RENUMBER_UNSAFE, ( + f"message_ts may be a Slack-issued timestamp and {where} it is the only " + "record of it; changing it fabricates an id Slack never issued" + ) + + +def mint_replacement_ts(original_ts: str, used: set[str], *, now_us: int, + modulus: int = EXPECTED_SLOT_MODULUS, + slot_id: int = REMEDIATION_WRITER_SLOT, + max_probes: int = 1000) -> str: + """Mint a ts-shaped id that collides with nothing, as close to the original as possible. + + Two properties matter, and they pull against each other: + + * **No collision.** The id lands in writer slot ``slot_id``, a residue class + no live minter uses, AND it is checked against ``used`` -- every + ``message_ts``, ``thread_ts`` and ``thread_decisions.thread_id`` already + present in that run, plus everything minted earlier in this same plan. So + it cannot collide with existing data, with another replacement, or with an + id a running engine goes on to mint. + * **Ordering.** ``message_ts`` doubles as the chronological key (0018) and + seeds ``posted_at`` (0019), so a replacement that jumps to "now" would + teleport an old message to the end of the conversation. The candidate is + therefore taken from the original's own microsecond slot: it lands at most + ``modulus * probes`` microseconds AFTER the id it replaces -- one slot in + the ordinary case, i.e. under 100µs. + + A ``message_ts`` that is not ts-shaped has no neighbourhood to stay in, so + those fall back to ``now_us``. Raises ``RuntimeError`` rather than looping + forever if ``max_probes`` consecutive slots are all taken. + """ + us = parse_ts_us(original_ts) + if us is None: + us = now_us + slot = us // modulus + for _ in range(max_probes): + candidate = format_us(slot * modulus + slot_id) + if candidate not in used and candidate != original_ts: + return candidate + slot += 1 + raise RuntimeError( + f"no free id in {max_probes} consecutive writer-{slot_id} slots after " + f"{original_ts!r}; refusing to renumber" + ) + + +@dataclass +class DuplicateGroup: + run_id: str + message_ts: str + rows: list[MessageRow] + #: rows elsewhere in the run whose thread_ts points at this message_ts + thread_reply_count: int = 0 + #: channel_ids those replies live in -- used to pick which row is the real root + thread_reply_channel_ids: set[str] = field(default_factory=set) + #: thread_decisions rows whose thread_id points at this message_ts + thread_decision_count: int = 0 + + kind: str = KIND_DIVERGENT + resolution: str = RESOLUTION_NEEDS_HUMAN + reason: str = "" + anchor_id: str | None = None + + @property + def row_count(self) -> int: + return len(self.rows) + + @property + def resolved(self) -> bool: + return self.resolution not in UNRESOLVED + + @property + def referenced(self) -> bool: + return bool(self.thread_reply_count or self.thread_decision_count) + + +def pick_anchor(rows: list[MessageRow], *, strategy: str, + reply_channel_ids: set[str]) -> MessageRow: + """Choose the row that KEEPS the original ``message_ts``. + + Order of preference: + + 1. The one row whose ts must not change (there is at most one, or the group + would have been refused). + 2. A row in a channel the thread replies live in. Replies point at a ts, not + at a row id, so whichever row keeps the ts inherits the replies; keeping + the ts on the row that actually started the thread is what makes those + pointers still mean something. + 3. Oldest row (``keep-latest``: newest), tie-broken on the primary key so the + choice is reproducible across runs and machines. + """ + unsafe = [r for r in rows if r.renumber_verdict == RENUMBER_UNSAFE] + if unsafe: + return min(unsafe, key=MessageRow.sort_key) + if reply_channel_ids: + in_reply_channel = [r for r in rows if r.channel_id in reply_channel_ids] + if in_reply_channel: + return min(in_reply_channel, key=MessageRow.sort_key) + if strategy == STRATEGY_KEEP_LATEST: + return max(rows, key=MessageRow.sort_key) + return min(rows, key=MessageRow.sort_key) + + +def plan_group(group: DuplicateGroup, *, strategy: str, used: set[str], now_us: int, + has_slack_columns: bool, writer_slots: dict[int, str], modulus: int) -> None: + """Classify ``group`` and fill in each row's action, in place. + + ``used`` is mutated: every id handed out is added, so two groups in the same + run can never be given the same replacement. + """ + for row in group.rows: + row.origin = classify_origin( + row, has_slack_columns=has_slack_columns, writer_slots=writer_slots, + modulus=modulus, + ) + row.renumber_verdict, row.renumber_reason = renumber_verdict( + row, row.origin, has_slack_columns=has_slack_columns + ) + row.action = ACTION_KEEP + row.new_message_ts = None + + payloads = {row.payload() for row in group.rows} + group.kind = KIND_REDUNDANT if len(payloads) == 1 else KIND_DIVERGENT + unsafe = [r for r in group.rows if r.renumber_verdict == RENUMBER_UNSAFE] + deletion_allowed = strategy in (STRATEGY_KEEP_EARLIEST, STRATEGY_KEEP_LATEST) + + if group.kind == KIND_REDUNDANT and deletion_allowed: + keep = (max if strategy == STRATEGY_KEEP_LATEST else min)( + group.rows, key=MessageRow.sort_key + ) + group.resolution = RESOLUTION_DELETE + group.anchor_id = keep.row_id + group.reason = ( + f"{group.row_count} rows equal on every column except id and created_at " + f"— one message logged {group.row_count} times. Deleting the " + f"{group.row_count - 1} extra row(s) loses no information." + ) + for row in group.rows: + row.action = ACTION_KEEP if row is keep else ACTION_DELETE + return + + if len(unsafe) <= 1: + anchor = pick_anchor( + group.rows, strategy=strategy, reply_channel_ids=group.thread_reply_channel_ids + ) + group.resolution = RESOLUTION_RENUMBER + group.anchor_id = anchor.row_id + if group.kind == KIND_REDUNDANT: + group.reason = ( + f"{group.row_count} byte-identical rows, all renumberable. Every row " + "is preserved, so nothing is lost — but the same message will then " + "appear twice in rebuilt history. --strategy keep-earliest would " + "delete the redundant copy instead, which is almost certainly what " + "you want." + ) + elif unsafe: + group.reason = ( + "rows differ, so no row may be deleted. Exactly one row's ts must not " + f"change ({unsafe[0].row_id}); it keeps the ts and the others are renumbered." + ) + else: + group.reason = ( + "rows differ, so no row may be deleted. Every ts here is safe to " + "change, so one row keeps the ts and the others are renumbered." + ) + for row in group.rows: + if row is anchor: + continue + row.action = ACTION_RENUMBER + row.new_message_ts = mint_replacement_ts(row.message_ts, used, now_us=now_us) + used.add(row.new_message_ts) + return + + if group.kind == KIND_REDUNDANT: + group.resolution = RESOLUTION_NEEDS_DELETE_STRATEGY + group.reason = ( + f"{len(unsafe)} of {group.row_count} rows carry a ts that must not change, " + "so renumbering cannot resolve this group. The rows ARE byte-identical, " + "so deleting the extras loses nothing: re-run with " + "--strategy keep-earliest." + ) + return + + group.resolution = RESOLUTION_NEEDS_HUMAN + group.reason = ( + f"{len(unsafe)} of {group.row_count} rows look Slack-born AND the rows differ. " + "No id may change and no row may be dropped, so one of these rows carries a " + "timestamp that is simply wrong and only a human can say which. " + "REFUSING to guess." + ) + + +def needs_human_advice(group: DuplicateGroup) -> list[str]: + """Exactly what an operator should look at for a refused group.""" + lines = [ + " What to check, in order:", + f" 1. Ask Slack which of these rows is real. For each row's channel_id, " + f"call conversations.history / conversations.replies with " + f"latest=oldest={group.message_ts} inclusive=true (scripts/backfill_slack_ts.py " + "does exactly this lookup, and a thread reply MUST be looked up with " + "conversations.replies — history does not return replies).", + " 2. At most one row can be the message Slack actually holds at that ts. " + "Any row Slack does not confirm has a fabricated or mis-copied ts.", + " 3. Compare the two bodies. If one row is empty or truncated, it is the " + "mis-logged one.", + ] + if group.referenced: + lines.append( + f" 4. {group.thread_reply_count} reply row(s) and " + f"{group.thread_decision_count} thread_decisions row(s) point at this ts. " + "Whichever row keeps it inherits them, so decide which row is the real " + "thread root before you touch anything." + ) + lines.append( + " Then fix that ONE row by hand (correct its message_ts, or delete it if " + "it is a mis-log) and re-run this tool." + ) + return lines + + +# --------------------------------------------------------------------------- # +# Database layer. +# --------------------------------------------------------------------------- # + +DUP_KEYS_SQL = f""" + SELECT simulation_run_id, message_ts, count(*) AS n + FROM {TABLE} + WHERE message_ts IS NOT NULL + GROUP BY simulation_run_id, message_ts + HAVING count(*) > 1 + ORDER BY simulation_run_id, message_ts +""" + +DUP_ROWS_SQL_TEMPLATE = """ + SELECT {cols} + FROM {table} m + WHERE m.message_ts IS NOT NULL + AND EXISTS ( + SELECT 1 FROM {table} d + WHERE d.simulation_run_id = m.simulation_run_id + AND d.message_ts = m.message_ts + AND d.id <> m.id + ) + ORDER BY m.simulation_run_id, m.message_ts, m.created_at, m.id +""" + + +def normalise_dsn(dsn: str) -> str: + """Force an async driver onto the DSN; this tool only speaks asyncpg.""" + if dsn.startswith("postgresql+"): + return dsn + if dsn.startswith("postgresql://"): + return "postgresql+asyncpg://" + dsn[len("postgresql://"):] + if dsn.startswith("postgres://"): + return "postgresql+asyncpg://" + dsn[len("postgres://"):] + return dsn + + +def redact_dsn(dsn: str) -> str: + """Mask only the password, keeping host/database legible (src/config.py does the same).""" + return re.sub(r"(://[^:/@]+:)[^@]*(@)", r"\1***\2", dsn) + + +async def fetch_schema(conn) -> dict[str, Any]: + from sqlalchemy import text + + cols = { + r[0] + for r in ( + await conn.execute( + text( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = current_schema() AND table_name = :t" + ), + {"t": TABLE}, + ) + ).all() + } + if not cols: + raise OperationalFailure( + f"table {TABLE!r} does not exist in the current schema — is this the " + "right database?" + ) + revision = None + has_version_table = ( + await conn.execute(text("SELECT to_regclass('alembic_version')")) + ).scalar() + if has_version_table: + revision = ( + await conn.execute(text("SELECT version_num FROM alembic_version LIMIT 1")) + ).scalar() + constraint_present = bool( + ( + await conn.execute( + text("SELECT 1 FROM pg_constraint WHERE conname = :c"), {"c": CONSTRAINT} + ) + ).scalar() + ) + thread_decisions = bool( + (await conn.execute(text("SELECT to_regclass('thread_decisions')"))).scalar() + ) + total, null_ts = ( + await conn.execute( + text(f"SELECT count(*), count(*) - count(message_ts) FROM {TABLE}") + ) + ).one() + return { + "columns": sorted(cols), + "alembic_revision": revision, + "has_content": "content" in cols, + "has_slack_ts": "slack_ts" in cols, + "constraint_present": constraint_present, + "has_thread_decisions": thread_decisions, + "total_rows": total, + "null_message_ts_rows": null_ts, + } + + +class OperationalFailure(RuntimeError): + """Something about the environment is wrong; not a verdict about the data.""" + + +async def load_groups(conn, columns: list[str], has_thread_decisions: bool) -> list[DuplicateGroup]: + from sqlalchemy import text + + quoted = ", ".join(f'm."{c}"' for c in columns) + sql = DUP_ROWS_SQL_TEMPLATE.format(cols=quoted, table=TABLE) + result = await conn.execute(text(sql)) + keys = list(result.keys()) + by_key: dict[tuple[str, str], DuplicateGroup] = {} + for record in result.all(): + values = dict(zip(keys, record, strict=True)) + run_id = str(values["simulation_run_id"]) + ts = str(values["message_ts"]) + row = MessageRow( + row_id=str(values["id"]), + run_id=run_id, + message_ts=ts, + created_at=values.get("created_at"), + columns=values, + ) + by_key.setdefault((run_id, ts), DuplicateGroup(run_id, ts, [])).rows.append(row) + + if not by_key: + return [] + + run_ids = sorted({run for run, _ in by_key}) + + # Rows elsewhere in these runs that point at a duplicated ts as their thread + # root. Runs are matched as text, not as a uuid[] bind: asyncpg has to infer + # the array's element type, and handing it Python uuid objects for an + # untyped placeholder is one more thing to get wrong for no benefit. + replies = ( + await conn.execute( + text( + f"SELECT simulation_run_id, thread_ts, count(*) AS n, " + f" array_agg(DISTINCT channel_id) AS channels " + f"FROM {TABLE} " + f"WHERE thread_ts IS NOT NULL AND simulation_run_id::text = ANY(:runs) " + f"GROUP BY simulation_run_id, thread_ts" + ), + {"runs": run_ids}, + ) + ).all() + for run_id, thread_ts, count, channels in replies: + group = by_key.get((str(run_id), str(thread_ts))) + if group is not None: + group.thread_reply_count = count + group.thread_reply_channel_ids = {str(c) for c in (channels or [])} + + if has_thread_decisions: + decisions = ( + await conn.execute( + text( + "SELECT simulation_run_id, thread_id, count(*) FROM thread_decisions " + "WHERE simulation_run_id::text = ANY(:runs) " + "GROUP BY simulation_run_id, thread_id" + ), + {"runs": run_ids}, + ) + ).all() + for run_id, thread_id, count in decisions: + group = by_key.get((str(run_id), str(thread_id))) + if group is not None: + group.thread_decision_count = count + + return [by_key[k] for k in sorted(by_key)] + + +async def load_used_ids(conn, run_ids: list[str], has_thread_decisions: bool) -> dict[str, set[str]]: + """Every id already spoken for in each affected run. + + ``message_ts`` is what the unique constraint covers, but ``thread_ts`` and + ``thread_decisions.thread_id`` are soft references BY VALUE: handing a + renumbered row an id that some reply already names as its thread root would + silently re-parent that reply. So all three are treated as taken. + """ + from sqlalchemy import text + + used: dict[str, set[str]] = defaultdict(set) + rows = ( + await conn.execute( + text( + f"SELECT simulation_run_id, message_ts FROM {TABLE} " + f"WHERE message_ts IS NOT NULL AND simulation_run_id::text = ANY(:runs) " + f"UNION " + f"SELECT simulation_run_id, thread_ts FROM {TABLE} " + f"WHERE thread_ts IS NOT NULL AND simulation_run_id::text = ANY(:runs)" + ), + {"runs": run_ids}, + ) + ).all() + for run_id, value in rows: + used[str(run_id)].add(str(value)) + if has_thread_decisions: + rows = ( + await conn.execute( + text( + "SELECT simulation_run_id, thread_id FROM thread_decisions " + "WHERE simulation_run_id::text = ANY(:runs)" + ), + {"runs": run_ids}, + ) + ).all() + for run_id, value in rows: + used[str(run_id)].add(str(value)) + return used + + +# --------------------------------------------------------------------------- # +# Reporting. +# --------------------------------------------------------------------------- # + +def _fmt_value(value: Any) -> str: + if isinstance(value, str) and len(value) > 80: + return f"{value[:77]!r}… ({len(value)} chars)" + return repr(value) + + +def print_group(group: DuplicateGroup, index: int, total: int, out) -> None: + print( + f"\n[{index}/{total}] run {group.run_id} message_ts {group.message_ts!r} " + f"{group.row_count} rows {group.kind.upper()}", + file=out, + ) + print( + f" inbound references: {group.thread_reply_count} thread repl(y/ies), " + f"{group.thread_decision_count} thread_decisions row(s)", + file=out, + ) + for row in group.rows: + marker = {ACTION_KEEP: "KEEP ", ACTION_RENUMBER: "RENUMBER", ACTION_DELETE: "DELETE "}[ + row.action + ] + print(f" {marker} id={row.row_id}", file=out) + origin = row.origin or Origin("unclassified", "not classified") + print(f" origin : {origin.verdict} — {origin.evidence}", file=out) + print( + f" renumber : {row.renumber_verdict} — {row.renumber_reason}", + file=out, + ) + if row.new_message_ts: + print( + f" new ts : {row.message_ts} -> {row.new_message_ts}", + file=out, + ) + detail = " ".join( + f"{name}={_fmt_value(value)}" + for name, value in sorted(row.columns.items()) + if name not in ("id", "simulation_run_id", "message_ts") + ) + print(f" columns : {detail}", file=out) + print(f" -> {group.resolution.upper()}: {group.reason}", file=out) + if group.resolution == RESOLUTION_NEEDS_HUMAN: + for line in needs_human_advice(group): + print(line, file=out) + + +def print_header(schema: dict[str, Any], dsn: str, strategy: str, apply: bool, + scheme_path: str, writer_slots: dict[int, str], out) -> None: + print("=" * 78, file=out) + print(f"remediate_duplicates — clear duplicate (simulation_run_id, message_ts) for {CONSTRAINT}", + file=out) + print("=" * 78, file=out) + print(f"database : {redact_dsn(dsn)}", file=out) + print(f"alembic revision : {schema['alembic_revision']}", file=out) + print( + f"schema : content={'yes' if schema['has_content'] else 'no'} " + f"slack_ts={'yes' if schema['has_slack_ts'] else 'no'} " + f"{CONSTRAINT}={'PRESENT' if schema['constraint_present'] else 'absent'}", + file=out, + ) + slots = ", ".join(f"{k}={v}" for k, v in sorted(writer_slots.items())) + print(f"id scheme : {scheme_path}", file=out) + print( + f" modulus={EXPECTED_SLOT_MODULUS} live slots [{slots}] " + f"remediation slot={REMEDIATION_WRITER_SLOT}", + file=out, + ) + print( + f"mode : {'APPLY (writes)' if apply else 'DRY RUN (READ ONLY transaction)'}", + file=out, + ) + print(f"strategy : {strategy}", file=out) + print( + f"{TABLE} : {schema['total_rows']} rows, of which " + f"{schema['null_message_ts_rows']} have message_ts IS NULL", + file=out, + ) + print( + " (NULL message_ts rows are exempt: the unique index is " + "NULLS DISTINCT,\n so any number of them coexist. This tool " + "never reads or writes them.)", + file=out, + ) + + +def group_to_json(group: DuplicateGroup) -> dict[str, Any]: + return { + "simulation_run_id": group.run_id, + "message_ts": group.message_ts, + "row_count": group.row_count, + "kind": group.kind, + "resolution": group.resolution, + "reason": group.reason, + "resolved": group.resolved, + "anchor_id": group.anchor_id, + "thread_reply_count": group.thread_reply_count, + "thread_decision_count": group.thread_decision_count, + "rows": [ + { + "id": row.row_id, + "origin": row.origin.verdict if row.origin else None, + "origin_evidence": row.origin.evidence if row.origin else None, + "renumber": row.renumber_verdict, + "renumber_reason": row.renumber_reason, + "action": row.action, + "new_message_ts": row.new_message_ts, + "columns": {k: _json_safe(v) for k, v in row.columns.items()}, + } + for row in group.rows + ], + } + + +def _json_safe(value: Any) -> Any: + if isinstance(value, datetime | uuid.UUID): + return str(value) + return value + + +# --------------------------------------------------------------------------- # +# Orchestration. +# --------------------------------------------------------------------------- # + +async def remediate(dsn: str, *, apply: bool, strategy: str, as_json: bool, + lock_timeout_ms: int) -> int: + from sqlalchemy import text + from sqlalchemy.ext.asyncio import create_async_engine + + modulus, writer_slots, scheme_path = load_id_scheme() + # With --json, stdout must stay parseable, so the human report goes to stderr. + out = sys.stderr if as_json else sys.stdout + engine = create_async_engine(dsn, pool_pre_ping=False) + try: + async with engine.begin() as conn: + if apply: + # One transaction for read, plan, write and re-verify. The lock + # blocks writers (INSERT/UPDATE/DELETE) but not readers, so the + # plan cannot be computed from rows another session is changing + # underneath us, and "no duplicates remain" is true as of commit. + await conn.execute(text(f"SET LOCAL lock_timeout = '{int(lock_timeout_ms)}ms'")) + await conn.execute(text(f"LOCK TABLE {TABLE} IN SHARE ROW EXCLUSIVE MODE")) + else: + # Belt and braces: a dry run cannot write even if this code is wrong. + await conn.execute(text("SET TRANSACTION READ ONLY")) + + schema = await fetch_schema(conn) + print_header(schema, dsn, strategy, apply, scheme_path, writer_slots, out) + groups = await load_groups(conn, schema["columns"], schema["has_thread_decisions"]) + + if not groups: + print( + f"\nNo duplicate (simulation_run_id, message_ts) groups. " + f"{CONSTRAINT} can be created as-is.", + file=out, + ) + payload = _envelope(schema, dsn, strategy, apply, [], None, EXIT_CLEAN) + if as_json: + print(json.dumps(payload, indent=2)) + return EXIT_CLEAN + + used = await load_used_ids( + conn, sorted({g.run_id for g in groups}), schema["has_thread_decisions"] + ) + now_us = time.time_ns() // 1000 + print( + f"\n{len(groups)} duplicate group(s) covering " + f"{sum(g.row_count for g in groups)} row(s):", + file=out, + ) + for index, group in enumerate(groups, start=1): + plan_group( + group, strategy=strategy, used=used[group.run_id], now_us=now_us, + has_slack_columns=schema["has_slack_ts"], writer_slots=writer_slots, + modulus=modulus, + ) + print_group(group, index, len(groups), out) + + unresolved = [g for g in groups if not g.resolved] + renumbers = [(g, r) for g in groups for r in g.rows if r.action == ACTION_RENUMBER] + deletes = [(g, r) for g in groups for r in g.rows if r.action == ACTION_DELETE] + print_summary(groups, unresolved, renumbers, deletes, apply, strategy, out) + + remaining = None + if apply: + if unresolved: + # Refuse the whole run rather than half-fixing the table: a + # partial fix still fails the migration, and it would have + # spent writes and a lock cycle to get there. + print( + f"\nREFUSING TO WRITE: {len(unresolved)} group(s) cannot be " + "resolved under this strategy (listed above). Nothing was " + "changed. Resolve those first — the migration will fail on " + "them regardless of what this tool fixes elsewhere.", + file=out, + ) + payload = _envelope( + schema, dsn, strategy, apply, groups, None, EXIT_REMAIN + ) + if as_json: + print(json.dumps(payload, indent=2)) + return EXIT_REMAIN + + # One statement per row, keyed on the primary key. Slower than a + # set-based UPDATE and deliberately so: every write is traceable + # to a row this tool printed, and a row that has changed under us + # cannot be caught by a broad predicate we no longer believe. + for _group, row in renumbers: + await conn.execute( + text(f"UPDATE {TABLE} SET message_ts = :new WHERE id = CAST(:id AS uuid)"), + {"new": row.new_message_ts, "id": row.row_id}, + ) + for _group, row in deletes: + await conn.execute( + text(f"DELETE FROM {TABLE} WHERE id = CAST(:id AS uuid)"), + {"id": row.row_id}, + ) + remaining = [ + {"simulation_run_id": str(r[0]), "message_ts": r[1], "row_count": r[2]} + for r in (await conn.execute(text(DUP_KEYS_SQL))).all() + ] + print( + f"\nApplied: {len(renumbers)} renumbered, {len(deletes)} deleted.", + file=out, + ) + if remaining: + print( + f"POST-CHECK FAILED: {len(remaining)} duplicate group(s) still " + f"present: {remaining}. Rolling back.", + file=out, + ) + raise _PostCheckFailed(remaining, schema, groups) + print( + "Post-check inside the same transaction: 0 duplicate groups remain. " + f"{CONSTRAINT} can now be created — run `alembic upgrade head`.", + file=out, + ) + code = EXIT_CLEAN + else: + code = EXIT_REMAIN if unresolved else EXIT_FOUND_DRY_RUN + + payload = _envelope(schema, dsn, strategy, apply, groups, remaining, code) + if as_json: + print(json.dumps(payload, indent=2)) + return code + except _PostCheckFailed as failure: + payload = _envelope( + failure.schema, dsn, strategy, apply, failure.groups, failure.remaining, EXIT_REMAIN + ) + if as_json: + print(json.dumps(payload, indent=2)) + return EXIT_REMAIN + finally: + await engine.dispose() + + +class _PostCheckFailed(Exception): + """Raised to force a rollback when duplicates survive the writes.""" + + def __init__(self, remaining, schema, groups): + super().__init__("duplicates remain after apply") + self.remaining = remaining + self.schema = schema + self.groups = groups + + +def print_summary(groups, unresolved, renumbers, deletes, apply, strategy, out) -> None: + counts: dict[str, int] = defaultdict(int) + for group in groups: + counts[group.resolution] += 1 + print("\n" + "-" * 78, file=out) + print("SUMMARY", file=out) + print(f" duplicate groups : {len(groups)}", file=out) + print(f" rows in those groups : {sum(g.row_count for g in groups)}", file=out) + for resolution in ( + RESOLUTION_DELETE, RESOLUTION_RENUMBER, RESOLUTION_NEEDS_DELETE_STRATEGY, + RESOLUTION_NEEDS_HUMAN, + ): + print(f" groups {resolution:<20}: {counts[resolution]}", file=out) + print(f" rows to renumber : {len(renumbers)}", file=out) + print(f" rows to delete : {len(deletes)}", file=out) + print("-" * 78, file=out) + if unresolved and not apply: + needs_delete = [ + g for g in unresolved if g.resolution == RESOLUTION_NEEDS_DELETE_STRATEGY + ] + needs_human = [g for g in unresolved if g.resolution == RESOLUTION_NEEDS_HUMAN] + print( + f"\n{len(unresolved)} group(s) would REMAIN, so the migration would still " + "fail. Nothing here is safe to automate:", + file=out, + ) + if needs_delete: + print( + f" * {len(needs_delete)} group(s) are byte-identical copies whose ts " + "must not change. Deleting the copies loses nothing — re-run with " + "--strategy keep-earliest to do it.", + file=out, + ) + if needs_human: + print( + f" * {len(needs_human)} group(s) need a human: two or more rows look " + "Slack-born and they are not identical. See the per-group notes above.", + file=out, + ) + elif not apply: + print( + f"\nDry run. Nothing was written (the transaction was READ ONLY). Re-run " + f"with --apply --strategy {strategy} to make these changes.", + file=out, + ) + + +def _envelope(schema, dsn, strategy, apply, groups, remaining, code) -> dict[str, Any]: + counts: dict[str, int] = defaultdict(int) + for group in groups: + counts[group.resolution] += 1 + return { + "tool": "remediate_duplicates", + "database": redact_dsn(dsn), + "alembic_revision": schema.get("alembic_revision"), + "schema": { + "has_content": schema.get("has_content"), + "has_slack_ts": schema.get("has_slack_ts"), + "constraint_present": schema.get("constraint_present"), + "total_rows": schema.get("total_rows"), + "null_message_ts_rows": schema.get("null_message_ts_rows"), + }, + "strategy": strategy, + "apply": apply, + "summary": { + "duplicate_groups": len(groups), + "rows_in_groups": sum(g.row_count for g in groups), + "by_resolution": dict(counts), + "rows_to_renumber": sum( + 1 for g in groups for r in g.rows if r.action == ACTION_RENUMBER + ), + "rows_to_delete": sum( + 1 for g in groups for r in g.rows if r.action == ACTION_DELETE + ), + "unresolved_groups": sum(1 for g in groups if not g.resolved), + }, + "duplicate_groups": [group_to_json(g) for g in groups], + "remaining_after_apply": remaining, + "exit_code": code, + } + + +class _Parser(argparse.ArgumentParser): + """argparse exits 2 on a usage error; 2 already means "duplicates found".""" + + def error(self, message: str) -> NoReturn: + self.print_usage(sys.stderr) + print(f"{self.prog}: error: {message}", file=sys.stderr) + raise SystemExit(EXIT_USAGE) + + +def build_parser() -> argparse.ArgumentParser: + parser = _Parser( + prog="remediate_duplicates", + description=( + "Clear duplicate (simulation_run_id, message_ts) rows in agent_messages " + "so migration 0019 can create uq_agent_messages_run_ts. Dry run unless " + "--apply is given." + ), + epilog=( + "exit codes: 0 clean / applied, 1 duplicates remain or would remain, " + "2 duplicates found in a dry run and all resolvable, 3 operational " + "failure, 64 usage error." + ), + ) + parser.add_argument( + "--database-url", + default=None, + help="Postgres DSN. Defaults to $DATABASE_URL.", + ) + parser.add_argument( + "--apply", action="store_true", + help="Actually write. Without this the tool only reports (READ ONLY transaction).", + ) + parser.add_argument( + "--strategy", choices=STRATEGIES, default=STRATEGY_RENUMBER, + help=( + "renumber (default, non-destructive): never delete a row; give the " + "duplicates new locally-minted ids where that is safe. " + "keep-earliest / keep-latest (destructive, opt-in): additionally DELETE " + "the redundant copies of byte-identical groups, keeping the oldest / " + "newest row. Divergent groups are still renumbered, never deleted, " + "under every strategy." + ), + ) + parser.add_argument( + "--json", action="store_true", dest="as_json", + help="Emit the machine-readable report on stdout (human report moves to stderr).", + ) + parser.add_argument( + "--lock-timeout-ms", type=int, default=int(os.environ.get("REMEDIATE_LOCK_TIMEOUT_MS", 10000)), + help=( + "How long --apply waits for the table lock before giving up (default " + "10000, matching ALEMBIC_LOCK_TIMEOUT_MS). 0 waits forever." + ), + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + dsn = args.database_url or os.environ.get("DATABASE_URL") + if not dsn: + print( + "ERROR: no database URL. Pass --database-url or set DATABASE_URL.", + file=sys.stderr, + ) + return EXIT_OPERATIONAL + try: + return asyncio.run( + remediate( + normalise_dsn(dsn), apply=args.apply, strategy=args.strategy, + as_json=args.as_json, lock_timeout_ms=args.lock_timeout_ms, + ) + ) + except SchemeError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return EXIT_OPERATIONAL + except OperationalFailure as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return EXIT_OPERATIONAL + except Exception as exc: # noqa: BLE001 — any DB failure is operational, not a verdict + print(f"ERROR: {type(exc).__name__}: {exc}", file=sys.stderr) + return EXIT_OPERATIONAL + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/migrate/run_migration.sh b/scripts/migrate/run_migration.sh new file mode 100755 index 0000000..43b3dce --- /dev/null +++ b/scripts/migrate/run_migration.sh @@ -0,0 +1,326 @@ +#!/usr/bin/env bash +# +# Guided production migration to alembic head 0023 (branch cohort-db-conversations). +# Supported starting points: 0018 (main before PR19), 0019, 0020 and 0021. +# 0021 is origin/main's own alembic head, so that is where a deployment tracking main is. +# +# READ docs/production-migration.md BEFORE RUNNING THIS. This script is the +# executable half of that runbook; the runbook explains *why* each step is where +# it is, which matters when a step fails. +# +# DEFAULT IS A REHEARSAL. Without --apply nothing is written: it runs the checks, +# prints the exact commands it *would* run, and tells you whether you are clear to +# proceed. That is deliberate — every other tool in this directory is dry-run by +# default and an operator who learns the convention from one must not be caught by +# another. +# +# ./scripts/migrate/run_migration.sh # rehearse, write nothing +# ./scripts/migrate/run_migration.sh --apply # back up, migrate, verify +# ./scripts/migrate/run_migration.sh --apply \ +# --backup-verified-elsewhere "nightly base backup + WAL, restore tested 2026-08-04" +# +# --backup-verified-elsewhere is the ONLY way to skip taking a dump, and it makes +# you write down what you are asserting instead. There is deliberately no bare +# "skip the backup check" flag: a safety tool must not accept a flag that quietly +# does nothing, and an operator must not be able to turn the check off without +# stating a reason that ends up in the log. +# +# EXIT CODES +# 0 rehearsal clear / migration applied and verified +# 1 BLOCKED — a check failed. Nothing was written. Fix and re-run. +# 2 rehearsal only: preflight raised warnings you should read. Nothing written. +# (In --apply mode warnings do not stop the run — you already chose to proceed — +# so a successful apply is still 0.) +# 3 operational failure (no DSN, unreachable DB, backup failed, lock timeout) +# 64 usage error +# +# WHAT THIS DOES NOT DO, on purpose: +# * It does not stop or start the agent/worker/web containers. Deciding when your +# traffic can pause is not a script's call, and a half-stopped deployment is +# worse than a refused one. It checks that nothing is holding a lock and tells +# you what to stop. +# * It does not run scripts/backfill_slack_ts.py. That one talks to Slack, needs a +# valid bot token in every affected channel, and its output needs a human to +# read. It is step 8 of the runbook, after this script. +# * It does not resolve duplicate (simulation_run_id, message_ts) rows. That is +# scripts/migrate/remediate_duplicates.py, which refuses the ambiguous cases on +# purpose. This script tells you to run it and stops. +set -euo pipefail + +EX_OK=0; EX_BLOCKED=1; EX_WARN=2; EX_OPERATIONAL=3; EX_USAGE=64 + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +APPLY=0 +TARGET="0024" +DSN="${DATABASE_URL:-}" +BACKUP_DIR="${MIGRATE_BACKUP_DIR:-backups}" +SVC="${MIGRATE_SERVICE:-app}" +PG_SVC="${MIGRATE_PG_SERVICE:-postgres}" +LOCK_TIMEOUT_MS="${ALEMBIC_LOCK_TIMEOUT_MS:-10000}" +BACKUP_VERIFIED_REASON="" +EXTRA_PREFLIGHT=() + +die_usage() { echo "ERROR: $*" >&2; echo "See docs/production-migration.md" >&2; exit "$EX_USAGE"; } + +while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=1; shift ;; + --target) TARGET="${2:?--target needs a revision}"; shift 2 ;; + --database-url) DSN="${2:?--database-url needs a DSN}"; shift 2 ;; + --backup-dir) BACKUP_DIR="${2:?--backup-dir needs a path}"; shift 2 ;; + --backup-verified-elsewhere) + BACKUP_VERIFIED_REASON="${2:?--backup-verified-elsewhere needs a reason}"; shift 2 ;; + # This flag used to be accepted and then silently ignored. Fail loudly rather + # than let anyone believe they turned the backup check off. + --skip-backup-check) + die_usage "--skip-backup-check no longer exists (it never did anything). + To proceed without letting this script take the dump, state what you are relying on: + --backup-verified-elsewhere \"nightly base backup + WAL, restore tested \"" ;; + # Print the header block by matching where it ENDS, not a hardcoded line number: + # the previous version said `2,45p` and had drifted into printing shell source, + # because editing the header silently invalidates a line count. + -h|--help) sed -n '2,/^set -euo pipefail/p' "$0" \ + | grep '^#' | sed 's/^# \{0,1\}//'; exit "$EX_OK" ;; + *) die_usage "unknown argument: $1" ;; + esac +done + +MODE="REHEARSAL (nothing will be written)" +[ "$APPLY" -eq 1 ] && MODE="APPLY (this will back up and migrate)" + +echo "==============================================================" +echo " coPI production migration -> $TARGET" +echo " mode: $MODE" +echo "==============================================================" + +# -------------------------------------------------------------------------- +# Step 1. The image must contain current code. +# +# Dockerfile does `pip install .`, which BAKES a copy of src/ into +# site-packages. For `python scripts/X.py`, CPython sets sys.path[0] to the +# script's own directory, so /app is NOT on the path and `import src` resolves to +# the baked copy — which in a stale container is days old. Every python step below +# therefore passes PYTHONPATH=/app, and this check proves it worked. +# -------------------------------------------------------------------------- +echo +echo "--- Step 1: the container is running current code ---" +if ! docker compose ps --status running --services 2>/dev/null | grep -qx "$SVC"; then + echo "BLOCKED: compose service '$SVC' is not running." >&2 + echo " docker compose up -d --build $SVC" >&2 + exit "$EX_OPERATIONAL" +fi +SRC_PATH="$(docker compose exec -T -e PYTHONPATH=/app "$SVC" \ + python -c 'import src; print(src.__file__)' 2>/dev/null | tr -d '\r')" +case "$SRC_PATH" in + /app/src/__init__.py) echo " PASS import src -> $SRC_PATH" ;; + *) echo "BLOCKED: with PYTHONPATH=/app, 'import src' resolved to '${SRC_PATH:-}'," >&2 + echo " not /app/src/__init__.py. The container would run stale code." >&2 + echo " docker compose up -d --build $SVC" >&2 + exit "$EX_BLOCKED" ;; +esac +if ! docker compose exec -T -e PYTHONPATH=/app "$SVC" \ + python -c 'from src.models import Cohort' >/dev/null 2>&1; then + echo "BLOCKED: /app/src has no Cohort model — the mounted source predates 0022." >&2 + exit "$EX_BLOCKED" +fi +echo " PASS /app/src carries the cohort models" + +# -------------------------------------------------------------------------- +# Step 2. Resolve the DSN, and say it out loud. +# +# alembic.ini defaults sqlalchemy.url to +# postgresql+asyncpg://copi:copi@localhost:5432/copi and env.py only overrides it +# when DATABASE_URL is set. A migration run with no DSN therefore targets whatever +# answers on localhost:5432. Measured on this machine: compose does not publish +# Postgres to the host, so a host-side run fails closed — but the bare hostname +# `postgres` resolves from the host to a PUBLIC IP (195.35.25.84) through a LAN +# search domain, so a DSN meant for in-container use points somewhere else entirely +# when it escapes. Never let it default, and always resolve it inside the container. +# -------------------------------------------------------------------------- +echo +echo "--- Step 2: target database ---" +if [ -z "$DSN" ]; then + echo "BLOCKED: no DSN. Pass --database-url or export DATABASE_URL." >&2 + echo " Refusing to let alembic.ini's localhost default choose the target." >&2 + exit "$EX_USAGE" +fi +echo " target: $(printf '%s' "$DSN" | sed -E 's#(//[^:]+):[^@]*@#\1:***@#')" + +run_py() { # run a repo python script inside the container with current code + docker compose exec -T -e PYTHONPATH=/app -e DATABASE_URL="$DSN" "$SVC" python "$@" +} + +# -------------------------------------------------------------------------- +# Step 3. Backup. Ordered BEFORE preflight so a blocked preflight still leaves you +# with a dump — and because migration 0019 is a one-way door: its downgrade drops +# agent_messages.content, i.e. every message body, silently and with exit 0. +# -------------------------------------------------------------------------- +echo +echo "--- Step 3: backup ---" +BACKUP_FILE="" +if [ -n "$BACKUP_VERIFIED_REASON" ]; then + echo " WARN backup asserted elsewhere: $BACKUP_VERIFIED_REASON" + EXTRA_PREFLIGHT+=(--backup-verified-elsewhere "$BACKUP_VERIFIED_REASON") +elif [ "$APPLY" -eq 0 ]; then + echo " (rehearsal) would write a custom-format dump into $BACKUP_DIR/" + EXTRA_PREFLIGHT+=(--backup-verified-elsewhere "rehearsal mode — no dump taken") +else + mkdir -p "$BACKUP_DIR" + DBNAME="$(printf '%s' "$DSN" | sed -E 's#.*/([^/?]+)(\?.*)?$#\1#')" + BACKUP_FILE="$BACKUP_DIR/${DBNAME}_pre${TARGET}_$(date +%Y%m%dT%H%M%S).dump" + echo " dumping $DBNAME -> $BACKUP_FILE" + # Dump to a file INSIDE the container, verify it there, then copy it out. + # + # Not `pg_dump … > host_file` piped back through `pg_restore -l /dev/stdin`: + # a custom-format archive needs random access to read its table of contents, and + # a pipe is not seekable, so that verification fails on a perfectly good dump. + # Caught by rehearsing this script end to end — it would have blocked every real + # migration at the backup step. + CTMP="/tmp/copi_migrate_$$.dump" + if ! docker compose exec -T "$PG_SVC" pg_dump -U copi -Fc -f "$CTMP" "$DBNAME"; then + echo "BLOCKED: pg_dump failed. Not migrating without a backup." >&2 + docker compose exec -T "$PG_SVC" rm -f "$CTMP" >/dev/null 2>&1 || true + exit "$EX_OPERATIONAL" + fi + # A dump whose table of contents cannot be read cannot be restored. This is the + # difference between having a backup and having a file. + if ! docker compose exec -T "$PG_SVC" pg_restore -l "$CTMP" >/dev/null 2>&1; then + echo "BLOCKED: pg_restore -l cannot read the dump — it is not restorable." >&2 + docker compose exec -T "$PG_SVC" rm -f "$CTMP" >/dev/null 2>&1 || true + exit "$EX_OPERATIONAL" + fi + TOC_N=$(docker compose exec -T "$PG_SVC" pg_restore -l "$CTMP" 2>/dev/null | grep -c '^[0-9]' || true) + docker compose cp "$PG_SVC:$CTMP" "$BACKUP_FILE" >/dev/null + docker compose exec -T "$PG_SVC" rm -f "$CTMP" >/dev/null 2>&1 || true + SZ=$(stat -c%s "$BACKUP_FILE" 2>/dev/null || echo 0) + if [ "$SZ" -lt 1024 ]; then + echo "BLOCKED: dump is only ${SZ} bytes — that is not a backup." >&2 + exit "$EX_OPERATIONAL" + fi + echo " PASS ${SZ} bytes on the host, ${TOC_N} restorable objects in the TOC" + EXTRA_PREFLIGHT+=(--backup-path "$BACKUP_FILE") +fi + +# -------------------------------------------------------------------------- +# Step 4. Preflight. Exit 1 here means STOP. +# -------------------------------------------------------------------------- +echo +echo "--- Step 4: preflight ---" +SNAP="${MIGRATE_SNAPSHOT:-$BACKUP_DIR/preflight_snapshot.json}" +mkdir -p "$(dirname "$SNAP")" +set +e +run_py scripts/migrate/preflight.py --target "$TARGET" --snapshot "$SNAP" \ + "${EXTRA_PREFLIGHT[@]}" +PF=$? +set -e +case "$PF" in + 0) echo " PASS preflight clear" ;; + 2) echo " WARN preflight raised warnings — read them above before continuing" ;; + *) echo "BLOCKED: preflight exited $PF. Nothing was written." >&2 + echo " If it reported duplicate (simulation_run_id, message_ts) rows:" >&2 + echo " docker compose exec -T -e PYTHONPATH=/app -e DATABASE_URL=\"\$DSN\" $SVC \\" >&2 + echo " python scripts/migrate/remediate_duplicates.py # dry run first" >&2 + exit "$EX_BLOCKED" ;; +esac + +if [ "$APPLY" -eq 0 ]; then + echo + echo "==============================================================" + echo " REHEARSAL COMPLETE — nothing was written." + echo " Re-run with --apply when your window is open." + # A rehearsal that raised warnings must not be indistinguishable, to a caller + # reading only the exit code, from one that was clear. Warnings here are things + # like "these rows will end up with content = ''" — real, unfixable, and worth an + # operator reading before the window rather than discovering after it. + if [ "$PF" -eq 2 ]; then + echo + echo " EXIT 2: preflight raised warnings. Scroll up and read them." + echo "==============================================================" + exit "$EX_WARN" + fi + echo "==============================================================" + exit "$EX_OK" +fi + +# -------------------------------------------------------------------------- +# Step 5. Migrate. One alembic command, so the whole chain is ONE transaction: +# a killed migration cannot leave a half-applied schema. +# -------------------------------------------------------------------------- +echo +echo "--- Step 5: alembic upgrade $TARGET (lock_timeout ${LOCK_TIMEOUT_MS}ms) ---" +set +e +docker compose exec -T -e PYTHONPATH=/app -e DATABASE_URL="$DSN" \ + -e ALEMBIC_LOCK_TIMEOUT_MS="$LOCK_TIMEOUT_MS" "$SVC" \ + python -m alembic upgrade "$TARGET" +MIG=$? +set -e +if [ "$MIG" -ne 0 ]; then + echo "BLOCKED: alembic exited $MIG." >&2 + echo " The chain runs in one transaction, so the database is unchanged — verify:" >&2 + echo " docker compose exec -T $PG_SVC psql -U copi -d -c 'select * from alembic_version'" >&2 + echo " A LockNotAvailableError means something held a lock on agent_messages." >&2 + echo " Stop the writers (docker stop -t 30 agent-run) and re-run." >&2 + exit "$EX_BLOCKED" +fi + +# -------------------------------------------------------------------------- +# Step 6. Confirm the commit by READING THE DATABASE. +# +# This step exists because of a real mistake made while building this tooling: a +# bad env.py change made every migration log "Running upgrade" and then silently +# roll the whole chain back, leaving no alembic_version table at all — and the +# log lines were mistaken for success. Alembic's own output is not evidence. +# -------------------------------------------------------------------------- +echo +echo "--- Step 6: confirm the commit landed ---" +STAMP="$(run_py -c " +import asyncio, os, sqlalchemy as sa +from sqlalchemy.ext.asyncio import create_async_engine +async def m(): + e = create_async_engine(os.environ['DATABASE_URL']) + async with e.connect() as c: + r = await c.execute(sa.text('select version_num from alembic_version')) + print((r.scalar() or 'NONE')) + await e.dispose() +asyncio.run(m())" 2>/dev/null | tr -d '\r')" +if [ "$STAMP" != "$TARGET" ]; then + echo "BLOCKED: alembic reported success but alembic_version is '${STAMP:-MISSING}', not $TARGET." >&2 + echo " Treat this as a silent rollback. Do NOT deploy code. Investigate env.py." >&2 + exit "$EX_BLOCKED" +fi +echo " PASS alembic_version = $STAMP (read back from the database)" + +# -------------------------------------------------------------------------- +# Step 7. Postflight: the schema, not the stamp. +# -------------------------------------------------------------------------- +echo +echo "--- Step 7: postflight ---" +set +e +run_py scripts/migrate/postflight.py --target "$TARGET" --snapshot "$SNAP" +POST=$? +set -e +if [ "$POST" -ne 0 ]; then + echo "BLOCKED: postflight exited $POST — the schema does not match $TARGET." >&2 + echo " Do NOT deploy application code. Restore path:" >&2 + echo " docs/production-migration.md, section 'If postflight fails'" >&2 + exit "$EX_BLOCKED" +fi +echo " PASS postflight verified" + +echo +echo "==============================================================" +echo " MIGRATION COMPLETE AND VERIFIED -> $TARGET" +[ -n "$BACKUP_FILE" ] && echo " backup: $BACKUP_FILE" +echo +echo " STILL TO DO, in this order (docs/production-migration.md steps 8-10):" +echo " 8. Repair the Slack mirror mapping on legacy rows:" +echo " docker compose exec -T -e PYTHONPATH=/app $SVC \\" +echo " python scripts/backfill_slack_ts.py # report first" +echo " docker compose exec -T -e PYTHONPATH=/app $SVC \\" +echo " python scripts/backfill_slack_ts.py --apply" +echo " Read its output. Exit 2 means rows were UNVERIFIED, not absent." +echo " 9. Deploy the application code, then restart app + worker." +echo " 10. Start agent-run last." +echo "==============================================================" diff --git a/scripts/mutate_cohorts.sh b/scripts/mutate_cohorts.sh new file mode 100755 index 0000000..525db37 --- /dev/null +++ b/scripts/mutate_cohorts.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# +# Mutation check for the cohort gate. Each mutant must be KILLED — at least one test +# must fail with it applied. A SURVIVING mutant means the suite does not actually test +# that behaviour, whatever its test names claim. +# +# This exists because four cohort tests were written that structurally could not fail, +# and each hid something. Two of the mutants below are the real defects those tests +# missed (M2, M6): both were found by a real multi-turn run, not by the suite. Running +# this after adding a cohort test is how you find that out in seconds instead. +# +# Offline: runs only the non-real_llm tests, so no API key and no spend. +# +# NOTHING IN THIS REPOSITORY IS EVER WRITTEN TO. +# Until 2026-08-04 this script mutated src/ IN PLACE and restored from a `.mutbak` +# copy — the same strategy scripts/mutate_system.sh documents as having been +# auto-reverted mid-run by a repo guard, silently corrupting three earlier agents' +# results (mutants reported as SURVIVING would in fact have been killed). It now uses +# mutate_system.sh's strategy instead, so the two harnesses share one isolation model: +# copy the tree into the container's /tmp, mutate the COPY, run pytest with the copy as +# its working directory, and PROVE — by importing `src` and checking `src.__file__` — +# that the copy is what is under test. That last check is not ceremony: `src` is also +# installed into site-packages in this image, so without it a run can exercise +# unmutated code and report every mutant as SURVIVED. +# +# Three guards, all lifted from mutate_system.sh: +# 1. provenance — `import src` from the copy must resolve inside the copy; +# 2. the mutant must still IMPORT — otherwise a SyntaxError fakes a kill, and a +# harness that cannot tell "the behaviour is tested" from "the file no longer +# parses" is not measuring anything. Reported as VOID, never as killed; +# 3. `git diff --quiet -- src/` before the first mutant and after the last. +# After every mutant the copy's file is restored from the read-only /app mount and +# `cmp`-checked, so one bad edit cannot silently pollute the rest of the run. +# +# THE INERT MUTANT IS NOT OPTIONAL. M0 below changes no behaviour (a docstring) and +# MUST SURVIVE. Without it, a selection that is red for any unrelated reason — a dead +# fixture, a migrated-away column, a leftover row — scores 9/9 and looks maximally +# sensitive when it is merely broken. It is listed FIRST so that failure is detected +# before any of the real mutants are believed. +# +# Usage: +# TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_test \ +# ./scripts/mutate_cohorts.sh +# +# Overridable env: +# TEST_DATABASE_URL throwaway asyncpg DSN (REQUIRED — these suites commit) +# MUTCOH_SERVICE compose service to exec into (default: app) +# MUTCOH_COPY_DIR where the mutated tree lives inside the container +# MUTCOH_LOGDIR where per-mutant pytest logs are kept (default: a mktemp dir) +# MUTCOH_KEEP_COPY set to 1 to leave the mutated tree behind for inspection +# +# `RUNNER` is gone. It used to be a whole pytest invocation pasted in as a string, +# which cannot express "run in the container but with the copy as cwd" — the override +# and the isolation strategy were mutually exclusive. Use MUTCOH_SERVICE / +# MUTCOH_COPY_DIR instead. +# +# MEASURED 2026-08-04, after the conversion: 9/9 real mutants killed, inert control +# survived, src/ clean. Same 9/9 the in-place harness reported, so no mutant moved — +# but that agreement is now backed by asserted provenance rather than assumed, and by +# an inert control the old harness did not have. The unmutated selection is 239 passed, +# checked separately, which is the other half of why the 9/9 means something. +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +: "${TEST_DATABASE_URL:?set TEST_DATABASE_URL to a throwaway database}" + +TESTS="tests/unit/test_cohort_isolation.py tests/integration/test_cohort_engine_live.py tests/integration/test_cohort_admin.py" +SVC="${MUTCOH_SERVICE:-app}" +COPY="${MUTCOH_COPY_DIR:-/tmp/mutcoh}" +LOGDIR="${MUTCOH_LOGDIR:-$(mktemp -d)}" +DC=(docker compose exec -T) + +# mkdir, because MUTCOH_LOGDIR is documented as overridable and an absent directory +# makes every `>"$log"` redirect fail — which the shell scores as a nonzero exit, i.e. +# as a kill, for every mutant including the inert control. Measured on mutate_system.sh, +# which had this bug: 6/6 "killed" and 0/4 inert survived. Only the inert control +# distinguished that from a real result. +mkdir -p "$LOGDIR" || { echo "ERROR: cannot create log dir $LOGDIR" >&2; exit 1; } + +# Deliberately NOT the live database, and asserted rather than assumed: the cohort +# engine and admin suites commit. +case "$TEST_DATABASE_URL" in + */copi|*/copi\?*) + echo "ERROR: TEST_DATABASE_URL points at the live 'copi' database. These suites" >&2 + echo "commit. Use a throwaway database." >&2 + exit 1 ;; +esac + +# --------------------------------------------------------------------------- +# Guard 3a: the working tree is never touched. Checked here, and again at the end. +# --------------------------------------------------------------------------- +if ! git diff --quiet -- src/; then + echo "ERROR: src/ has uncommitted changes." >&2 + echo "This script does not edit src/ — it mutates a copy inside the container — but a" >&2 + echo "dirty tree means the copy would carry changes that are not the mutant, so every" >&2 + echo "result below would be unattributable. Commit or stash first." >&2 + exit 1 +fi + +# file ~~ exact source substring ~~ replacement ~~ what it breaks +# The delimiter is ~~ and not | because one target contains a pipe +# (`gates[aid] = mates | unrestricted`) — the very line whose mutation is M2. +# `\n` in the FROM/TO fields is a newline (see the applier below). +MUTANTS=( +# The inert control runs FIRST: if it does not survive, no number below is a score. +"src/services/cohorts.py~~ \"\"\"Counts for logging and the admin banner.\"\"\"~~ \"\"\"Counts for logging and for the admin banner. [INERT EDIT]\"\"\"~~M0 INERT docstring — MUST SURVIVE" +"src/services/cohorts.py~~gates[aid] = set() if isolate_uncohorted else None~~gates[aid] = set()~~M1 open-policy uncohorted agent is silenced instead of unrestricted" +"src/services/cohorts.py~~gates[aid] = mates | unrestricted~~gates[aid] = mates~~M2 the open-policy asymmetry (a REAL defect the suite missed)" +"src/services/cohorts.py~~effective = cohort_count if live_members is None else live_members~~effective = cohort_count~~M3 preflight counts cohorts, not live members, so an empty cohort silences the roster" +"src/agent/message_log.py~~ if not entry.is_bot:~~ if entry.sender_agent_id is None:~~M4 the human bypass keys on a NULL agent_id, so an unattributable bot row leaks" +"src/agent/message_log.py~~ if entry.visibility == VISIBILITY_COLLAB_PRIVATE:~~ if False:~~M5 the private-channel exemption is dead" +# M6 was pinned to `visibility=self._resolve_channel_visibility(channel),` — the +# keyword argument inside the LogEntry(...) call. d311170 hoisted the resolution out of +# that call so the chunk loop could reuse one value, and the old target stopped existing. +# Re-pointed 2026-08-04 at the assignment, which is the same defect: every chunk of every +# outbound post is then stamped public. Nothing detected the drift for five days because +# nothing re-ran this script; when it was re-run it reported ERROR rather than a false +# kill, which is the one thing the old harness did get right. +"src/agent/simulation.py~~ visibility = self._resolve_channel_visibility(channel)~~ visibility = VISIBILITY_PUBLIC~~M6 outbound messages are never stamped collab_private (a REAL defect the suite missed)" +"src/agent/simulation.py~~ if thread.grandfathered:\n continue~~ if False:\n continue~~M7 a grandfathered thread keeps reactive priority" +"src/agent/simulation.py~~ if self._reactive_streak < settings.max_consecutive_reactive_turns:~~ if True:~~M8 the fairness valve never closes" +"src/agent/simulation.py~~ if target_id == agent.agent_id or target_id in allowed:~~ if True:~~M9 the outbound tag strip never strips" +) + +# --------------------------------------------------------------------------- +# Build the mutable copy inside the container and PROVE it is what runs. +# --------------------------------------------------------------------------- +cleanup() { + if [ "${MUTCOH_KEEP_COPY:-0}" = "1" ]; then + echo "(left the mutated tree at ${SVC}:${COPY} — MUTCOH_KEEP_COPY=1)" + else + "${DC[@]}" "$SVC" rm -rf "$COPY" >/dev/null 2>&1 + fi +} +trap cleanup EXIT + +echo "building a throwaway copy of the tree at ${SVC}:${COPY} (the repo is never written to)" +if ! "${DC[@]}" "$SVC" sh -c " + rm -rf '$COPY' && mkdir -p '$COPY' && + tar -C /app \ + --exclude=./.git --exclude=./.venv-test --exclude=./mutants --exclude=./build \ + --exclude=./logs --exclude=./.hypothesis --exclude=./.pytest_cache \ + --exclude=./.ruff_cache --exclude=./.playwright-mcp --exclude=__pycache__ \ + -cf - . | tar -C '$COPY' -xf - +" 2>/dev/null; then + echo "ERROR: could not copy /app into $COPY inside the '$SVC' container." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Guard 1: provenance. +# --------------------------------------------------------------------------- +prov=$("${DC[@]}" -w "$COPY" "$SVC" python -c "import src; print(src.__file__)" 2>/dev/null | tr -d '\r') +case "$prov" in + "$COPY"/src/__init__.py) echo "provenance OK: pytest will import $prov" ;; + *) + echo "ERROR: from $COPY, 'import src' resolves to '${prov:-}', not" >&2 + echo "$COPY/src/__init__.py. The mutants would not be under test. Refusing to run." >&2 + exit 1 ;; +esac + +echo "logs: $LOGDIR" +echo + +# --------------------------------------------------------------------------- +# Run +# --------------------------------------------------------------------------- +fail=0 killed=0 survived=0 void=0 broken_inert=0 inert_ok=0 n=0 +declare -a SURVIVORS=() + +for m in "${MUTANTS[@]}"; do + file="${m%%~~*}"; rest="${m#*~~}" + from="${rest%%~~*}"; rest="${rest#*~~}" + to="${rest%%~~*}"; label="${rest#*~~}" + n=$((n + 1)) + short="${label%% *}" + + inert=0; [[ "$label" == *INERT* ]] && inert=1 + + # --- apply the mutation to the COPY -------------------------------------------------- + if ! "${DC[@]}" -e "FROM=$from" -e "TO=$to" "$SVC" python - "$COPY/$file" <<'PY' 2>&1 +import os, pathlib, sys +p = pathlib.Path(sys.argv[1]) +s = p.read_text() +frm = os.environ["FROM"].replace("\\n", "\n") +to = os.environ["TO"].replace("\\n", "\n") +if frm not in s: + sys.stderr.write(f"mutation target not found in {p}:\n{frm!r}\n"); sys.exit(1) +if s.count(frm) != 1: + sys.stderr.write(f"target occurs {s.count(frm)} times in {p}; it must be unique\n") + sys.exit(1) +p.write_text(s.replace(frm, to, 1)) +PY + then + echo "ERROR $label — target string not found (or not unique); the code moved," >&2 + echo " fix this script rather than the test." >&2 + fail=1 + "${DC[@]}" "$SVC" cp -- "/app/$file" "$COPY/$file" >/dev/null 2>&1 + continue + fi + + # --- Guard 2: the mutant must still import ------------------------------------------ + # A SyntaxError makes every test in the selection error out, which is indistinguishable + # from a kill unless it is checked for. Derived from the path so a new mutant in a new + # file is covered without editing this line. + mod=$(printf '%s' "${file%.py}" | tr '/' '.') + if ! "${DC[@]}" -w "$COPY" "$SVC" python -c "import $mod" >/dev/null 2>&1; then + echo "VOID $label — the mutated module does not import, so a kill here would" >&2 + echo " only mean 'the file no longer parses'. Fix the replacement text." >&2 + void=$((void + 1)); fail=1 + "${DC[@]}" "$SVC" cp -- "/app/$file" "$COPY/$file" >/dev/null 2>&1 + continue + fi + + log="$LOGDIR/$(printf '%02d' "$n")-${short}.log" + # -x: stop at the first failure. The killer's name is what the report needs, and the + # inert control above is what makes attributing it sound. + if "${DC[@]}" -e "TEST_DATABASE_URL=$TEST_DATABASE_URL" -w "$COPY" "$SVC" \ + sh -c "python -m pytest $TESTS -q -x -rf -m 'not real_llm' -p no:cacheprovider" \ + >"$log" 2>&1; then + if [ "$inert" -eq 1 ]; then + echo "survived (expected) $label" + inert_ok=$((inert_ok + 1)) + else + echo "SURVIVED $label" + SURVIVORS+=("$label") + survived=$((survived + 1)); fail=1 + fi + else + killer=$(grep -m1 '^FAILED ' "$log" | sed 's/^FAILED //') + if [ "$inert" -eq 1 ]; then + echo "KILLED AN INERT MUTANT $label" >&2 + echo " -> ${killer:-see $log}" >&2 + echo " The selection is failing for a reason that is NOT the mutation, so" >&2 + echo " every other number in this run is meaningless." >&2 + broken_inert=$((broken_inert + 1)); fail=1 + else + echo "killed $label" + echo " by ${killer:-}" + killed=$((killed + 1)) + fi + fi + + # --- restore the copy from the pristine mount, and verify it ------------------------ + "${DC[@]}" "$SVC" cp -- "/app/$file" "$COPY/$file" >/dev/null 2>&1 + if ! "${DC[@]}" "$SVC" cmp -s "/app/$file" "$COPY/$file"; then + echo "ERROR: $COPY/$file no longer matches /app/$file; the copy is polluted and" >&2 + echo "every result after this point is unattributable. Stopping." >&2 + exit 1 + fi +done + +# --------------------------------------------------------------------------- +# Guard 3b: the working tree must be exactly as we found it. +# --------------------------------------------------------------------------- +if ! git diff --quiet -- src/; then + echo >&2 + echo "ERROR: src/ is dirty. This script never writes to src/, so something else did." >&2 + echo "Inspect 'git diff -- src/' before doing anything else." >&2 + exit 1 +fi + +echo +echo "killed ${killed}/$((killed + survived + void)) real mutants" +echo "inert controls: ${inert_ok}/$((inert_ok + broken_inert)) survived (all of them must)" +echo "src/ clean: yes" + +if [ "$broken_inert" -gt 0 ]; then + echo >&2 + echo "AN INERT MUTANT WAS KILLED. Read nothing else in this run as a score: the" >&2 + echo "selection is red for an unrelated reason, which makes a broken suite look" >&2 + echo "maximally sensitive. Fix that first, then re-run." >&2 +fi +if [ "${#SURVIVORS[@]}" -gt 0 ]; then + echo >&2 + echo "SURVIVING MUTANTS — each is a behaviour the suite does not protect:" >&2 + for s in "${SURVIVORS[@]}"; do echo " - $s" >&2; done + echo "Add the test that kills it. Do not weaken the mutant." >&2 +fi +if [ "$fail" -eq 0 ]; then + echo "all mutants killed and the inert control survived — the cohort suite has teeth" +fi +exit "$fail" diff --git a/scripts/mutate_slack_mirror.sh b/scripts/mutate_slack_mirror.sh new file mode 100755 index 0000000..bfa3861 --- /dev/null +++ b/scripts/mutate_slack_mirror.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# +# Mutation check for the DB<->Slack mirror. Each mutant must be KILLED by the live +# Slack tier. A SURVIVING mutant means the live tests do not actually test that +# behaviour — which is the whole reason they exist, since the offline suite runs with +# NullTransport and cannot see the mirror at all (Rule S2). +# +# Needs the live workspace, including all three probe bot tokens: the lifecycle tests +# compare the bots against each other. Slower and more expensive than +# scripts/mutate_cohorts.sh — each mutant is a full live run against Slack. +# +# TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a6 \ +# ./scripts/mutate_slack_mirror.sh +# +# --------------------------------------------------------------------------------------- +# CONVERTED 2026-08-04, when live workspace credentials first became available. Until then +# this script edited src/ IN PLACE via a .mutbak copy, and could not be run even once — so +# it was left alone deliberately, on the grounds that rewriting a measurement harness you +# cannot execute converts a known weakness into an unknown one. All four defects its own +# header listed are now fixed, and the result was run three times: +# +# 1. the tree is copied into the container's /tmp and the COPY is mutated, with pytest +# run from the copy as its working directory; +# 2. provenance is asserted — `import src` from the copy must resolve to +# "$COPY/src/__init__.py". `src` is ALSO installed into site-packages in this image, +# so without this a run can exercise unmutated code and report every mutant as +# SURVIVED; +# 3. the mutated module must still import, so a SyntaxError cannot fake a kill; +# 4. `git diff --quiet -- src/` is asserted before the first mutant and after the last; +# 5. `\n` in the FROM/TO fields becomes a real newline, matching the other two harnesses; +# 6. per-mutant output is kept and a kill NAMES the test that killed it. Discarding +# output made a mutant that "killed" because the workspace was unreachable +# indistinguishable from a real kill. +# +# S4 is the inert control and MUST SURVIVE — a tier without one scores 100% precisely when +# it is broken. mutate_system.sh once printed "killed 6/6" beside "inert controls: 0/4 +# survived" because its log directory did not exist and every redirect failed; the inert +# control was the only signal. Hence the mkdir -p below. +# --------------------------------------------------------------------------------------- +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +: "${SLACK_TEST_WORKSPACE:?live workspace credentials required}" +: "${TEST_DATABASE_URL:?set TEST_DATABASE_URL to a throwaway database}" + +SVC="${MUTMIRROR_SERVICE:-app}" +COPY="${MUTMIRROR_COPY_DIR:-/tmp/mutmirror}" +LOGDIR="${MUTMIRROR_LOGDIR:-$(mktemp -d)}" +mkdir -p "$LOGDIR" + +ENVARGS=() +for v in SLACK_TEST_WORKSPACE SLACK_TEST_PI_USER_ID SLACK_TEST_TEAM_ID \ + SLACK_TEST_BOT_TOKEN_SU SLACK_TEST_BOT_TOKEN_CRAVATT SLACK_TEST_BOT_TOKEN_WISEMAN \ + ANTHROPIC_API_KEY LIVE_API_TESTS TEST_DATABASE_URL; do + [ -n "${!v:-}" ] && ENVARGS+=(-e "$v=${!v}") +done +TESTS="tests/integration/test_slack_mirror_live.py tests/integration/test_slack_lifecycle_live.py" + +if ! git diff --quiet -- src/; then + echo "ERROR: src/ has uncommitted changes. Commit or stash first — a mutation run" >&2 + echo "against a dirty tree cannot be attributed to the mutants." >&2 + exit 1 +fi + +# file ~~ exact source substring ~~ replacement ~~ what it breaks +MUTANTS=( +"src/agent/simulation.py~~ slack_ts=slack_ts,~~ slack_ts=None,~~S1 the mirror mapping is never recorded on an outbound post" +"src/agent/simulation.py~~ return root.slack_ts~~ return thread_ts~~S2 a canonical id is handed to Slack (a93d136)" +"src/agent/slack_client.py~~ self._client = None~~ pass~~S3 a dead token still reports is_connected (the bug found by T11)" +"src/agent/slack_client.py~~ last_exc: SlackApiError | None = None~~ last_exc = None # noqa~~S4 sanity: this edit is inert and MUST survive" +) + +echo "building a throwaway copy of the tree at ${SVC}:${COPY} (the repo is never written to)" +docker compose exec -T "$SVC" bash -c " + rm -rf '$COPY' && mkdir -p '$COPY' && + cd /app && tar --exclude=./.git --exclude=./.venv-test -cf - . | tar -C '$COPY' -xf - +" >/dev/null 2>&1 || { echo "ERROR: could not copy /app into $COPY" >&2; exit 1; } + +prov=$(docker compose exec -T -w "$COPY" "$SVC" python -c "import src; print(src.__file__)" 2>/dev/null | tr -d '\r') +if [ "$prov" != "$COPY/src/__init__.py" ]; then + echo "ERROR: from $COPY, 'import src' resolves to '${prov:-}', not" >&2 + echo "$COPY/src/__init__.py. The mutants would not be under test. Refusing to run." >&2 + docker compose exec -T "$SVC" rm -rf "$COPY" >/dev/null 2>&1 + exit 1 +fi +echo "provenance OK: pytest will import $prov" + +cleanup() { + if [ "${MUTMIRROR_KEEP_COPY:-0}" = "1" ]; then + echo "(left the mutated tree at ${SVC}:${COPY} — MUTMIRROR_KEEP_COPY=1)" + else + docker compose exec -T "$SVC" rm -rf "$COPY" >/dev/null 2>&1 + fi +} +trap cleanup EXIT INT TERM + +run_selection() { # $1 = log file + docker compose exec -T -w "$COPY" "${ENVARGS[@]}" "$SVC" \ + python -m pytest $TESTS -q -m live_slack -p no:cacheprovider > "$1" 2>&1 +} + +echo; echo "=== baseline (unmutated copy) ===" +if ! run_selection "$LOGDIR/baseline.log"; then + echo "ERROR: the unmutated copy is RED — no mutant result below would mean anything." >&2 + grep -E "^FAILED|^ERROR|passed|failed" "$LOGDIR/baseline.log" | tail -5 >&2 + exit 1 +fi +grep -E "passed|failed" "$LOGDIR/baseline.log" | tail -1 +echo + +fail=0; killed=0; i=0 +for m in "${MUTANTS[@]}"; do + i=$((i+1)) + file="${m%%~~*}"; rest="${m#*~~}" + from="${rest%%~~*}"; rest="${rest#*~~}" + to="${rest%%~~*}"; label="${rest#*~~}" + inert=0; [[ "$label" == S4* ]] && inert=1 + + if ! docker compose exec -T -e "FROM=$from" -e "TO=$to" "$SVC" python - "$COPY/$file" <<'PY' +import os, pathlib, sys +p = pathlib.Path(sys.argv[1]); s = p.read_text() +frm = os.environ["FROM"].replace("\\n", "\n") +to = os.environ["TO"].replace("\\n", "\n") +n = s.count(frm) +if n != 1: + sys.stderr.write(f"expected exactly 1 occurrence in {p.name}, found {n}: {frm!r}\n") + sys.exit(1) +p.write_text(s.replace(frm, to, 1)) +PY + then + echo "ERROR $label — target not found or not unique; the code moved" >&2 + docker compose exec -T "$SVC" cp "/app/$file" "$COPY/$file" >/dev/null 2>&1 + fail=1; continue + fi + + mod="${file#src/}"; mod="src.${mod%.py}"; mod="${mod//\//.}" + if ! docker compose exec -T -w "$COPY" "$SVC" python -c "import $mod" >/dev/null 2>&1; then + echo "VOID $label — the mutated module does not import; result discarded" >&2 + docker compose exec -T "$SVC" cp "/app/$file" "$COPY/$file" >/dev/null 2>&1 + fail=1; continue + fi + + log="$LOGDIR/m$i.log" + if run_selection "$log"; then + if [ "$inert" -eq 1 ]; then + echo "survived (expected) $label [$(grep -oE '[0-9]+ passed' "$log" | tail -1)]" + killed=$((killed+1)) + else + echo "SURVIVED $label [$(grep -oE '[0-9]+ passed' "$log" | tail -1)]"; fail=1 + fi + else + if [ "$inert" -eq 1 ]; then + echo "KILLED AN INERT MUTANT $label — the tier is flaky or broken, not sensitive" >&2 + grep -E "^FAILED|^ERROR" "$log" | head -3 >&2 + fail=1 + else + killers=$(grep -oE "^FAILED [^ ]+" "$log" | sed 's/^FAILED //' | head -3 | tr '\n' ' ') + echo "killed $label" + echo " by: ${killers:-no FAILED line — inspect $log}" + killed=$((killed+1)) + fi + fi + docker compose exec -T "$SVC" cp "/app/$file" "$COPY/$file" >/dev/null 2>&1 +done + +echo +git diff --quiet -- src/ || { echo "FATAL: src/ was modified — results are void" >&2; exit 1; } +echo "repo clean check: src/ untouched" +echo "killed ${killed}/${#MUTANTS[@]}" +echo "logs: $LOGDIR" +[ "$fail" -eq 0 ] && echo "the live Slack mirror tier has teeth" || echo "SURVIVING OR VOID MUTANTS" >&2 +exit "$fail" diff --git a/scripts/mutate_system.sh b/scripts/mutate_system.sh new file mode 100755 index 0000000..50ffe9d --- /dev/null +++ b/scripts/mutate_system.sh @@ -0,0 +1,406 @@ +#!/usr/bin/env bash +# +# Mutation check for the subsystems T1–T11 claim to protect: ORCID, PubMed/NCBI, the +# job-queue worker, the profile pipeline, the public graph, onboarding/impersonation/ +# profile export, the agent page, and GrantBot's FOA dedup. +# +# Each mutant must be KILLED — at least one test in the named selection must fail with it +# applied. A SURVIVING mutant means the suite does not actually test that behaviour, +# whatever its test names claim. Same discipline as scripts/mutate_cohorts.sh (9/9) and +# scripts/mutate_slack_mirror.sh (4/4), and the same `~~` field delimiter, chosen there +# because one mutation target contains a `|` and kept here because several contain `~`-free +# SQL with pipes and quotes of both kinds. +# +# As of 2026-08-04 mutate_cohorts.sh shares this file's isolation strategy — it was +# converted from in-place editing to copy+provenance, and its 9/9 was re-measured under +# the new strategy and held. mutate_slack_mirror.sh still edits src/ in place; it needs +# live Slack credentials, so it could not be re-verified after a rewrite and was left +# alone with a header warning rather than silently changed. +# +# THE INERT MUTANTS ARE NOT OPTIONAL. Every tier below carries one edit that changes no +# behaviour (a docstring, a comment, a log string) and MUST SURVIVE. Without it a tier +# that is broken for any unrelated reason — a dead credential, a migrated-away column, a +# leftover row — scores 100% and looks sensitive when it is merely failing. Each tier's +# inert mutant is listed FIRST so a broken tier is detected before any money is spent on +# it. +# +# NOTHING IN THIS REPOSITORY IS EVER WRITTEN TO. +# Three agents previously applied mutations by editing src/ in place. A repo guard +# auto-reverted them mid-run and silently corrupted the results: mutants reported as +# SURVIVING would in fact have been killed. So this script copies the tree into the +# container's /tmp, mutates the COPY, runs pytest with the copy as its working directory, +# and proves — by importing `src` and checking `src.__file__` — that the copy is what is +# under test. `git diff --quiet -- src/` is asserted before the first mutant and after the +# last one. +# +# Usage: +# # offline tiers only (free, no third-party calls): +# ./scripts/mutate_system.sh +# +# # + the live ORCID / NCBI / grants.gov tiers (free, but real HTTP): +# LIVE_API_TESTS=1 ./scripts/mutate_system.sh +# +# # + the profile-pipeline tier (real Anthropic tokens, ~7 calls total): +# LIVE_API_TESTS=1 ANTHROPIC_API_KEY=sk-ant-... ./scripts/mutate_system.sh +# +# A tier whose credentials are absent is reported as `skipped`, never as `killed`. +# +# Cost control: every mutant runs against ONLY the test file (and often only the single +# test) that is supposed to kill it, never the whole suite. That is what keeps the +# Anthropic spend at ~7 calls and the NCBI traffic inside the 3 req/s anonymous policy. +# +# MEASURED 2026-08-04, offline tiers only, no credentials present: +# +# killed 6/6 real mutants M4, M5 (worker); M7 (graph); M8, M9 (onboarding); +# M10 (agentpage) +# inert controls 4/4 survived M12c, M12e, M12f, M12g +# 11 skipped for credentials orcid: M12a, M1, M1b +# pubmed: M12b, M2, M3 +# pipeline: M12d, M6, M6b +# grantbot: M12h, M11 +# src/ clean, exit 0 +# +# NO REAL MUTANT SURVIVED ANY TIER THAT COULD BE RUN. The list below is therefore not a +# list of survivors; it is the standing record for the two mutants this script's own +# tiers cannot judge without credentials, plus the resolution of one that used to survive. +# Do not weaken any of them. +# +# M1b UNMEASURED as of 2026-08-04 — its tier (orcid) needs LIVE_API_TESTS=1, which was +# not available, so it reported `skipped`. It was last measured as SURVIVING on +# 2026-07-31 and nothing has changed tests/live_api/test_orcid_live.py since, so +# treat it as still open: fetch_orcid_profile hardcoded to "Josiah Carberry" +# survives that file. Its only defence against a constant name is the dated +# `"Carberry" in prof["name"]` assertion, which a hardcode of the expected value +# satisfies. Nothing in the live tier compares the parsed name against the record +# it came from, and the docstring's claimed control ("the parser must NOT return +# the same thing for a different id") is not implemented — the id it checks is +# copied from the argument, not parsed. Measured then: the PRE-EXISTING contract +# test tests/contract/test_orcid_contract.py:: +# test_fetch_orcid_profile_falls_back_to_orcid_when_no_name DOES kill it, so this +# is a gap in the new tier rather than in the repo. Re-run with LIVE_API_TESTS=1 +# before claiming it either way. +# +# M6 RESOLVED 2026-08-04. _validate_profile hardwired to `return True` is now KILLED +# by tests/characterization/test_profile_pipeline_gm.py (3 failed: +# stores_the_retry_not_the_rejected_first_synthesis, +# marks_a_profile_that_fails_validation_twice, +# rerun_that_fails_validation_keeps_the_stored_profile), and M6b (always False) by +# 7 — so the validator's effect is now visible in BOTH directions, which it was +# not before. An inert docstring edit on the same function survived the same +# selection (11 passed), so those kills are the mutation and not a red suite. +# Fix 2 (d311170) is what closed it: the return value now gates step 9 instead of +# being computed and discarded. The old note here claimed M6 survived the entire +# offline suite; that stopped being true and nothing updated it. +# +# CAVEAT, so this is not misread: the kill comes from the OFFLINE characterization +# file, not from this script's `pipeline` tier, which is +# tests/integration/test_profile_pipeline_live.py and still needs LIVE_API_TESTS=1 +# plus ANTHROPIC_API_KEY. M6/M6b therefore still report `skipped` in a +# credential-free run — see the 2026-08-04 measurement above. The evidence was +# produced with this script's own copy+provenance pattern (tree copied into the +# container, `src.__file__` asserted under the copy, the mutated module +# import-checked), not by editing src/. If you want this harness to see M6 by +# itself, the characterization file has to join a tier whose CREDS are "". +# +# 2026-08-04, the Slack chokepoint mutants — 8515f65's control was a PROVENANCE ARTIFACT. +# That commit recorded, as the most important line in its report, that the four +# chokepoint mutants "run against the offline selection ALL SURVIVED, at exactly 1093", +# and flagged the suspiciously round figure as needing reproduction before belief. It has +# now been reproduced, and the claim does not hold: against +# tests/unit/test_slack_client_contract.py + tests/unit/test_transport.py (81 passed +# unmutated) all four are KILLED, with an inert docstring edit on the same file surviving. +# The pagination mutant was additionally run against the WHOLE offline suite, which is the +# selection the 1093 figure came from: 9 failed / 1158 passed / 120 skipped, the same 9 +# tests. So the full suite kills it too — and today's collection is 1158, not 1093, which +# is a second reason that figure describes a run that was not measuring what it claimed. +# +# pagination _paginate returns after page 1 ............... 9 failed +# ts-ordering the pre-fix list(reversed(...)) in +# _conversation_messages ........................ 4 failed +# (the stronger `key=_by_ts, reverse=True` variant: 7 failed) +# splitting split_for_slack never splits (`if True: return [text]`) 7 failed +# thread_ts normalise normalize_inbound_message's self-reference test dead 5 failed +# INERT control _conversation_messages docstring reworded .... survived +# +# So Fix 4's offline tests do protect all four mechanisms; the earlier "all survived" was +# the failure mode this script's provenance check exists to catch — unmutated code under +# test, every mutant falsely surviving. Which is also why the exact-1093 count was the +# tell: a selection that never loaded the mutant cannot move. +# +# Overridable env: +# TEST_DATABASE_URL throwaway asyncpg DSN (default: the copi_a3 scratch database) +# MUTSYS_SERVICE compose service to exec into (default: app) +# MUTSYS_COPY_DIR where the mutated tree lives inside the container +# MUTSYS_LOGDIR where per-mutant pytest logs are kept (default: a mktemp dir) +# MUTSYS_KEEP_COPY set to 1 to leave the mutated tree behind for inspection +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +TEST_DATABASE_URL="${TEST_DATABASE_URL:-postgresql+asyncpg://copi:copi@postgres:5432/copi_a3}" +SVC="${MUTSYS_SERVICE:-app}" +COPY="${MUTSYS_COPY_DIR:-/tmp/mutsys}" +LOGDIR="${MUTSYS_LOGDIR:-$(mktemp -d)}" +DC=(docker compose exec -T) + +# mkdir, because MUTSYS_LOGDIR is documented as overridable and this script did not +# create it. Measured 2026-08-04: pass a path that does not exist and every `>"$log"` +# redirect fails, which the shell scores as a nonzero exit — i.e. as a KILL — for every +# mutant, inert controls included. The run reported "killed 6/6 real mutants" and +# "inert controls: 0/4 survived", and only that second line distinguished it from a +# perfect score. Any earlier run of this script made with MUTSYS_LOGDIR set to a +# nonexistent directory reported every mutant as killed and is void. +mkdir -p "$LOGDIR" || { echo "ERROR: cannot create log dir $LOGDIR" >&2; exit 1; } + +# Deliberately NOT the live database, and asserted rather than assumed: several of these +# suites commit (the worker tests need another connection to see the write, so they cannot +# use the rolled-back session fixture). +case "$TEST_DATABASE_URL" in + */copi|*/copi\?*) + echo "ERROR: TEST_DATABASE_URL points at the live 'copi' database. These suites" >&2 + echo "commit. Use a throwaway database." >&2 + exit 1 ;; +esac + +# --------------------------------------------------------------------------- +# Rule 1: the working tree is never touched. Check before, and again at the end. +# --------------------------------------------------------------------------- +if ! git diff --quiet -- src/; then + echo "ERROR: src/ has uncommitted changes." >&2 + echo "This script does not edit src/ — it mutates a copy inside the container — but a" >&2 + echo "dirty tree means the copy would carry changes that are not the mutant, so every" >&2 + echo "result below would be unattributable. Commit or stash first." >&2 + exit 1 +fi + +# --------------------------------------------------------------------------- +# Tiers: the selection each mutant is judged against, and what it costs. +# +# CREDS: "" = offline; "live" = needs LIVE_API_TESTS=1; "live+llm" = also real Anthropic. +# --------------------------------------------------------------------------- +declare -A TIER_SELECT=( + [orcid]="tests/live_api/test_orcid_live.py" + [pubmed_tool]="tests/live_api/test_pubmed_live.py -k test_ncbi_get_sends_the_required_tool_and_email_parameters" + [pubmed_doi]="tests/live_api/test_pubmed_live.py -k test_reconcile_pub_doi_separates_a_real_match_from_a_near_miss" + [pubmed_both]="tests/live_api/test_pubmed_live.py -k 'test_ncbi_get_sends_the_required_tool_and_email_parameters or test_reconcile_pub_doi_separates_a_real_match_from_a_near_miss'" + [worker]="tests/integration/test_worker.py" + [pipeline]="tests/integration/test_profile_pipeline_live.py -k test_t41_one_real_orcid_becomes_a_stored_profile_grounded_in_its_works" + [graph]="tests/integration/test_public_graph.py" + [onboarding]="tests/integration/test_onboarding_flow.py" + [agentpage]="tests/integration/test_agent_page.py" + [grantbot]="tests/integration/test_grantbot_live.py -m 'not real_llm' -k 'test_claim_foa_is_the_dedup_primitive or test_the_claim_not_the_prefilter_is_what_stops_a_repost'" +) +declare -A TIER_CREDS=( + [orcid]="live" [pubmed_tool]="live" [pubmed_doi]="live" [pubmed_both]="live" + [worker]="" [pipeline]="live+llm" [graph]="" [onboarding]="" + [agentpage]="" [grantbot]="live" +) + +# tier ~~ file ~~ exact source substring ~~ replacement ~~ label +# +# `\n` in the FROM/TO fields is a newline (see the applier below). Entries containing a +# double quote are single-quoted here and vice versa; no entry needs both. +MUTANTS=( +# --- ORCID (T1) ------------------------------------------------------------------------ +'orcid~~src/services/orcid.py~~ """Extract name, affiliation, and email from ORCID record."""~~ """Extract the name, affiliation and email from an ORCID record. [INERT EDIT]"""~~M12a INERT docstring — MUST SURVIVE' +'orcid~~src/services/orcid.py~~ result["name"] = f"{given} {family}".strip() or orcid_id~~ result["name"] = "Ada Lovelace"~~M1 fetch_orcid_profile returns a constant name instead of parsing person.name' +# M1b is the same defect as M1 with the constant chosen to equal today's expected value. +# It is the difference between "the test reads the record" and "the test restates the +# answer". SURVIVES the live tier (see KNOWN SURVIVORS above). +'orcid~~src/services/orcid.py~~ result["name"] = f"{given} {family}".strip() or orcid_id~~ result["name"] = "Josiah Carberry"~~M1b the same hardcode, set to the value the test pins (probes whether the assertion is derived from the live record or merely restated)' +# --- PubMed / NCBI (T2) ---------------------------------------------------------------- +'pubmed_both~~src/services/pubmed.py~~ """Make a rate-limited, identified GET request to NCBI."""~~ """Make a rate-limited, identified GET request to NCBI E-utilities. [INERT EDIT]"""~~M12b INERT docstring — MUST SURVIVE' +"pubmed_doi~~src/services/pubmed.py~~ if assigned.lower() == auth.lower():~~ if True:~~M2 reconcile_pub_doi always reports a match, so a PMID keeps whatever DOI it arrived with" +'pubmed_tool~~src/services/pubmed.py~~ params.setdefault("tool", _NCBI_TOOL)~~ pass # tool= no longer sent~~M3 _ncbi_get stops identifying itself to NCBI (throttle, then IP block)' +# --- worker (T5) ----------------------------------------------------------------------- +'worker~~src/worker/main.py~~ logger.info("Job %s completed", job.id)~~ logger.info("Job %s has completed", job.id)~~M12c INERT log string — MUST SURVIVE' +"worker~~src/worker/main.py~~ .with_for_update(skip_locked=True)~~ .with_for_update()~~M4 claim_job drops SKIP LOCKED, so a worker pool serialises behind the slowest job" +'worker~~src/worker/main.py~~ if job.type == "generate_profile":~~ job.status = "completed"; job.completed_at = datetime.now(timezone.utc); await db.commit()\n if job.type == "generate_profile":~~M5 the job is marked completed and committed BEFORE the work is dispatched' +# --- profile pipeline (T4) ------------------------------------------------------------- +"pipeline~~src/services/profile_pipeline.py~~ Validate synthesized profile fields.~~ Validate the synthesized profile fields. [INERT EDIT]~~M12d INERT docstring — MUST SURVIVE" +"pipeline~~src/services/profile_pipeline.py~~def _validate_profile(profile: dict[str, Any]) -> bool:~~def _validate_profile(profile: dict[str, Any]) -> bool:\n return True~~M6 _validate_profile always returns True, so no profile is ever rejected" +"pipeline~~src/services/profile_pipeline.py~~def _validate_profile(profile: dict[str, Any]) -> bool:~~def _validate_profile(profile: dict[str, Any]) -> bool:\n return False~~M6b the same function always returns False — the paired control for M6, which shows whether the tier can see validation's effect in EITHER direction" +# --- public graph (T9) ----------------------------------------------------------------- +"graph~~src/routers/public.py~~ -- The agent-only proposal for a thread is the FIRST one the bots~~ -- [INERT EDIT] the agent-only proposal for a thread is the FIRST one the bots~~M12e INERT SQL comment inside the mutated query — MUST SURVIVE" +"graph~~src/routers/public.py~~ AND origin_visibility = 'public'\n AND decided_at >= :decided_floor{window_end_clause}~~ AND decided_at >= :decided_floor{window_end_clause}~~M7 the pairs CTE stops filtering on origin_visibility, so collab_private proposals reach the public graph" +# --- onboarding / impersonation / profile export (T7) ---------------------------------- +"onboarding~~src/dependencies.py~~ # Impersonation: admin can view as another user~~ # Impersonation [INERT EDIT]: an admin can view the site as another user~~M12f INERT comment — MUST SURVIVE" +"onboarding~~src/dependencies.py~~ if impersonate_id and session_user.is_admin:~~ if impersonate_id:~~M8 copi-impersonate is honoured for non-admins — any logged-in user can become any other user" +'onboarding~~src/services/profile_export.py~~ path = PROFILES_DIR / f"{agent_id}.md"~~ if profile.private_profile_md:\n lines.append(profile.private_profile_md)\n path = PROFILES_DIR / f"{agent_id}.md"~~M9 the PUBLIC profile export appends private_profile_md' +# --- agent page (T8) ------------------------------------------------------------------- +'agentpage~~src/routers/agent_page.py~~ "Ignoring duplicate reopen of proposal %s by %s "~~ "Ignoring a duplicate reopen of proposal %s by %s "~~M12g INERT log string — MUST SURVIVE' +"agentpage~~src/routers/agent_page.py~~ if already_reviewed is not None:~~ if False:~~M10 the reopen idempotency guard is gone, so a replayed POST migrates the thread twice" +# --- GrantBot (T10) -------------------------------------------------------------------- +'grantbot~~src/agent/grantbot.py~~ """Undo a claim when the Slack post itself failed, so a later run can retry."""~~ """Undo a claim when the Slack post failed, so a later run retries. [INERT EDIT]"""~~M12h INERT docstring — MUST SURVIVE' +"grantbot~~src/agent/grantbot.py~~ return result.rowcount == 1~~ return True~~M11 _claim_foa always reports the claim as won, so two runs post the same FOA" +) + +# --------------------------------------------------------------------------- +# Build the mutable copy inside the container and PROVE it is what runs. +# --------------------------------------------------------------------------- +cleanup() { + if [ "${MUTSYS_KEEP_COPY:-0}" = "1" ]; then + echo "(left the mutated tree at ${SVC}:${COPY} — MUTSYS_KEEP_COPY=1)" + else + "${DC[@]}" "$SVC" rm -rf "$COPY" >/dev/null 2>&1 + fi +} +trap cleanup EXIT + +echo "building a throwaway copy of the tree at ${SVC}:${COPY} (the repo is never written to)" +if ! "${DC[@]}" "$SVC" sh -c " + rm -rf '$COPY' && mkdir -p '$COPY' && + tar -C /app \ + --exclude=./.git --exclude=./.venv-test --exclude=./mutants --exclude=./build \ + --exclude=./logs --exclude=./.hypothesis --exclude=./.pytest_cache \ + --exclude=./.ruff_cache --exclude=./.playwright-mcp --exclude=__pycache__ \ + -cf - . | tar -C '$COPY' -xf - +" 2>/dev/null; then + echo "ERROR: could not copy /app into $COPY inside the '$SVC' container." >&2 + exit 1 +fi + +# Provenance. `src` is ALSO installed into site-packages in this image, so without this +# check a run could silently be testing /usr/local/.../src or /app/src — i.e. exercising +# unmutated code and reporting every mutant as SURVIVED. That is the failure mode this +# whole script exists to avoid, so it is asserted rather than assumed. +prov=$("${DC[@]}" -w "$COPY" "$SVC" python -c "import src; print(src.__file__)" 2>/dev/null | tr -d '\r') +case "$prov" in + "$COPY"/src/__init__.py) echo "provenance OK: pytest will import $prov" ;; + *) + echo "ERROR: from $COPY, 'import src' resolves to '${prov:-}', not" >&2 + echo "$COPY/src/__init__.py. The mutants would not be under test. Refusing to run." >&2 + exit 1 ;; +esac + +echo "logs: $LOGDIR" +echo + +# --------------------------------------------------------------------------- +# Run +# --------------------------------------------------------------------------- +fail=0 killed=0 survived=0 skipped=0 broken_inert=0 inert_ok=0 n=0 +declare -a SURVIVORS=() + +for m in "${MUTANTS[@]}"; do + tier="${m%%~~*}"; rest="${m#*~~}" + file="${rest%%~~*}"; rest="${rest#*~~}" + from="${rest%%~~*}"; rest="${rest#*~~}" + to="${rest%%~~*}"; label="${rest#*~~}" + n=$((n + 1)) + short="${label%% *}" + + inert=0; [[ "$label" == *INERT* ]] && inert=1 + select="${TIER_SELECT[$tier]}" + creds="${TIER_CREDS[$tier]}" + + # --- credentials gate: a tier we cannot run is `skipped`, never `killed` ------------- + envargs=(-e "TEST_DATABASE_URL=$TEST_DATABASE_URL") + case "$creds" in + live|live+llm) + if [ -z "${LIVE_API_TESTS:-}" ]; then + echo "skipped $label (needs LIVE_API_TESTS=1)"; skipped=$((skipped + 1)); continue + fi + envargs+=(-e "LIVE_API_TESTS=$LIVE_API_TESTS") ;; + esac + if [ "$creds" = "live+llm" ]; then + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "skipped $label (needs ANTHROPIC_API_KEY — this tier spends real tokens)" + skipped=$((skipped + 1)); continue + fi + envargs+=(-e "ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY") + fi + + # --- apply the mutation to the COPY -------------------------------------------------- + if ! "${DC[@]}" -e "FROM=$from" -e "TO=$to" "$SVC" python - "$COPY/$file" <<'PY' 2>&1 +import os, pathlib, sys +p = pathlib.Path(sys.argv[1]) +s = p.read_text() +frm = os.environ["FROM"].replace("\\n", "\n") +to = os.environ["TO"].replace("\\n", "\n") +if frm not in s: + sys.stderr.write(f"mutation target not found in {p}:\n{frm!r}\n"); sys.exit(1) +if s.count(frm) != 1: + sys.stderr.write(f"target occurs {s.count(frm)} times in {p}; it must be unique\n") + sys.exit(1) +p.write_text(s.replace(frm, to, 1)) +PY + then + echo "ERROR $label — target string not found (or not unique); the code moved," >&2 + echo " fix this script rather than the test." >&2 + fail=1 + "${DC[@]}" "$SVC" cp -- "/app/$file" "$COPY/$file" >/dev/null 2>&1 + continue + fi + + log="$LOGDIR/$(printf '%02d' "$n")-${short}.log" + # -x: stop at the first failure. The killer's name is what the report needs, and a + # narrow selection plus a per-tier inert control is what makes attributing it sound. + if "${DC[@]}" "${envargs[@]}" -w "$COPY" "$SVC" \ + sh -c "python -m pytest $select -q -x -rf -p no:cacheprovider" >"$log" 2>&1; then + if [ "$inert" -eq 1 ]; then + echo "survived (expected) $label" + killed=$((killed + 1)); inert_ok=$((inert_ok + 1)) + else + echo "SURVIVED $label" + SURVIVORS+=("$label") + survived=$((survived + 1)); fail=1 + fi + else + killer=$(grep -m1 '^FAILED ' "$log" | sed 's/^FAILED //') + if [ "$inert" -eq 1 ]; then + echo "KILLED AN INERT MUTANT $label" >&2 + echo " -> ${killer:-see $log}" >&2 + echo " This tier is failing for a reason that is NOT the mutation, so" >&2 + echo " every other number for it is meaningless." >&2 + broken_inert=$((broken_inert + 1)); fail=1 + else + echo "killed $label" + echo " by ${killer:-}" + killed=$((killed + 1)) + fi + fi + + # --- restore the copy from the pristine mount, and verify it ------------------------ + "${DC[@]}" "$SVC" cp -- "/app/$file" "$COPY/$file" >/dev/null 2>&1 + if ! "${DC[@]}" "$SVC" cmp -s "/app/$file" "$COPY/$file"; then + echo "ERROR: $COPY/$file no longer matches /app/$file; the copy is polluted and" >&2 + echo "every result after this point is unattributable. Stopping." >&2 + exit 1 + fi +done + +# --------------------------------------------------------------------------- +# The working tree must be exactly as we found it. +# --------------------------------------------------------------------------- +if ! git diff --quiet -- src/; then + echo >&2 + echo "ERROR: src/ is dirty. This script never writes to src/, so something else did." >&2 + echo "Inspect 'git diff -- src/' before doing anything else." >&2 + exit 1 +fi + +real_killed=$((killed - inert_ok)) +real_total=$((real_killed + survived)) +echo +echo "killed ${real_killed}/${real_total} real mutants" +echo "inert controls: ${inert_ok}/$((inert_ok + broken_inert)) survived (all of them must)" +echo "${skipped} skipped for missing credentials" +echo "src/ clean: yes" + +if [ "$broken_inert" -gt 0 ]; then + echo >&2 + echo "AN INERT MUTANT WAS KILLED. Read nothing else in this run as a score: the tier it" >&2 + echo "belongs to is red for an unrelated reason, which makes a broken suite look" >&2 + echo "maximally sensitive. Fix that first, then re-run." >&2 +fi +if [ "${#SURVIVORS[@]}" -gt 0 ]; then + echo >&2 + echo "SURVIVING MUTANTS — each is a behaviour the suite does not protect:" >&2 + for s in "${SURVIVORS[@]}"; do echo " - $s" >&2; done + echo "Add the test that kills it. Do not weaken the mutant." >&2 +fi +if [ "$fail" -eq 0 ]; then + echo "every judged mutant was killed and every inert control survived" +fi +exit "$fail" diff --git a/scripts/provision_slack_bots.py b/scripts/provision_slack_bots.py index 6065dba..5ed8817 100644 --- a/scripts/provision_slack_bots.py +++ b/scripts/provision_slack_bots.py @@ -63,11 +63,14 @@ # Shared provisioning helpers (transport-only; no heavy deps so this stays # importable on the host). See src/services/slack_provisioning.py. from src.services.slack_provisioning import ( + BOT_SCOPES, create_app, exchange_code, - lookup_team_id as _slack_lookup_team_id, rotate_config_token, ) +from src.services.slack_provisioning import ( + lookup_team_id as _slack_lookup_team_id, +) # --------------------------------------------------------------------------- # Constants @@ -109,7 +112,7 @@ def load_roster() -> list[dict]: if not ROSTER_PATH.exists(): raise RuntimeError( f"{ROSTER_PATH} not found. Generate it in the container first:\n" - f" docker exec copi-python-app-1 python scripts/export_agent_roster.py" + f" docker compose exec app python scripts/export_agent_roster.py" ) roster = json.loads(ROSTER_PATH.read_text()) return [r for r in roster if r.get("status") in PROVISIONABLE_STATUSES] @@ -249,6 +252,13 @@ def main(): help="Slack workspace team ID (e.g. T012AB3CD) to pin OAuth URLs to the right workspace. " "Auto-detected from an existing bot token if not provided.", ) + parser.add_argument( + "--omit-scope", action="append", default=[], metavar="AGENT_ID:SCOPE", + help="Create AGENT_ID's app without SCOPE. Repeatable. The scope set is fixed " + "at manifest time — Slack's consent screen has no per-scope choice — so " + "this is the only way to install a bot deliberately missing one. The live " + "tier needs it for wiseman: --omit-scope wiseman:groups:write", + ) parser.add_argument( "--only", nargs="+", metavar="AGENT_ID", help="Only provision these agent_id(s), e.g. --only good. " @@ -288,6 +298,14 @@ def main(): if not lab.get("has_token") and lab["id"] not in tokenized ] + omit: dict[str, set[str]] = {} + for spec in args.omit_scope: + aid, _, scope = spec.partition(":") + if not aid or not scope: + console.print(f"[red]--omit-scope needs AGENT_ID:SCOPE, got {spec!r}[/red]") + raise SystemExit(2) + omit.setdefault(aid.lower(), set()).add(scope) + if args.only: only = {a.lower() for a in args.only} unknown = only - {lab["id"].lower() for lab in roster} @@ -320,15 +338,24 @@ def main(): config_token = existing_env.get("SLACK_CONFIG_TOKEN", "").strip() refresh_token = existing_env.get("SLACK_CONFIG_REFRESH_TOKEN", "").strip() - if not config_token: - console.print("\n[bold red]SLACK_CONFIG_TOKEN is not set in .env[/bold red]") + # EITHER credential is enough. Requiring SLACK_CONFIG_TOKEN here was a bug: the + # rotation below derives a fresh access token from the refresh token and + # overwrites config_token unconditionally, so a refresh-token-only .env — the + # normal state after a rotation, since rotation replaces both — was rejected + # for want of a value the script was about to discard anyway. + if not config_token and not refresh_token: + console.print( + "\n[bold red]Neither SLACK_CONFIG_TOKEN nor SLACK_CONFIG_REFRESH_TOKEN " + "is set in .env[/bold red]" + ) console.print( " 1. Open https://api.slack.com/apps in a browser\n" " 2. Click 'Your App Configuration Tokens'\n" " 3. Click 'Generate Token' for your workspace\n" - " 4. Copy the token (xoxe-...) and refresh token into .env:\n" - " SLACK_CONFIG_TOKEN=xoxe-...\n" - " SLACK_CONFIG_REFRESH_TOKEN=xoxe-...\n" + " 4. Copy BOTH values into .env. Note the prefixes differ:\n" + " SLACK_CONFIG_TOKEN=xoxe.xoxp-... (access token, ~12h life)\n" + " SLACK_CONFIG_REFRESH_TOKEN=xoxe-1-... (refresh token, single use)\n" + " The refresh token alone is sufficient — it mints the access token.\n" ) sys.exit(1) @@ -342,6 +369,25 @@ def main(): except Exception as exc: console.print(f"[yellow]Token rotation failed ({exc}); using existing token.[/yellow]") + # Rotation can fail with a still-empty config_token — a refresh token that was + # revoked, expired, or already spent (they are single use, so a value left in + # .env after someone rotated it elsewhere is dead). Stop here rather than + # calling apps.manifest.create with an empty Authorization header, which fails + # with an opaque Slack error that says nothing about the real cause. + if not config_token: + console.print( + "\n[bold red]No usable config token.[/bold red] The refresh token in " + f"{args.env_file} did not rotate, and SLACK_CONFIG_TOKEN is empty." + ) + console.print( + " Config refresh tokens are SINGLE USE: whoever rotated it last holds the\n" + " only live one, and a copy left behind in .env is already dead.\n" + " Generate a fresh pair at https://api.slack.com/apps -> " + "'Your App Configuration Tokens'\n" + " and replace BOTH values in .env." + ) + sys.exit(1) + # ----------------------------------------------------------------------- # 3. Start OAuth callback server (before app creation so URLs work immediately) # ----------------------------------------------------------------------- @@ -389,7 +435,14 @@ def _oauth_url(app: dict) -> str: failed_count = 0 for i, lab in enumerate(missing): try: - app = create_app(config_token, lab["id"], lab["name"], lab["pi"], redirect_uri) + dropped = omit.get(lab["id"].lower(), set()) + scopes = [x for x in BOT_SCOPES if x not in dropped] if dropped else None + if dropped: + console.print(f" [yellow]omitting scope(s) {sorted(dropped)} for {lab['id']}[/yellow]") + app = create_app( + config_token, lab["id"], lab["name"], lab["pi"], redirect_uri, + scopes=scopes, + ) created.append(app) # Persist immediately (0600) so an interruption mid-run doesn't # lose the client_secret we just minted. @@ -437,12 +490,13 @@ def _oauth_url(app: dict) -> str: if done < total: outstanding = [a["bot_name"] for a in created if a["agent_id"] not in _CallbackHandler.received] console.print(f"[yellow]Still missing: {', '.join(outstanding)}[/yellow]") - console.print(f"Re-run with [bold]--skip-create[/bold] to retry without recreating the apps.") + console.print("Re-run with [bold]--skip-create[/bold] to retry without recreating the apps.") else: if STATE_FILE.exists(): STATE_FILE.unlink() - console.print(f"[green]All done! Restart the agent container to pick up the new tokens.[/green]") - console.print(" docker rm -f agent-run") + console.print("[green]All done! Restart the agent container to pick up the new tokens.[/green]") + console.print(" docker stop -t 30 agent-run # SIGTERM so the engine flushes") + console.print(" docker rm agent-run") console.print(" docker compose up -d --build app worker") console.print(" docker compose --profile agent run -d --name agent-run agent python -m src.agent.main --budget 0") diff --git a/scripts/setup_inbound_email.py b/scripts/setup_inbound_email.py new file mode 100644 index 0000000..957a0e2 --- /dev/null +++ b/scripts/setup_inbound_email.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +""" +Check (and optionally provision) the AWS/DNS infrastructure for inbound email +replies — the review+TOKEN@reply.copi.science flow. + +Background (investigation of 2026-08-11) +----------------------------------------- +The reply-by-email review flow shipped in code but its infrastructure was +never provisioned on prod. Every layer was missing, so PI replies bounced and +nothing was processed: + + 1. DNS: reply.copi.science had NO MX record (replies never reached AWS). + 2. S3: the copi-inbound-email bucket did not exist. + 3. SES: no receipt rule delivered mail for the reply domain to S3. + 4. IAM: copi-ec2-ses-role had send-only perms (no S3 read/delete for polling). + 5. Env: ENABLE_INBOUND_EMAIL was unset, so the worker never polled anyway. + +This script verifies each layer (--check, the default) and can create the AWS +pieces (--provision). DNS records must be added at the registrar by hand; the +script prints exactly what to add. + +Prerequisites +------------- +Run from a machine/profile with ADMIN AWS credentials (SES receipt rules, S3 +bucket creation, IAM read). The EC2 instance role is NOT sufficient — that is +finding #4 above. + +Usage +----- + # Report the state of every layer, change nothing: + python scripts/setup_inbound_email.py --check + + # Create bucket + policy + receipt rule set/rule, then print DNS + IAM steps: + python scripts/setup_inbound_email.py --provision + + # Non-default names: + python scripts/setup_inbound_email.py --check \ + --region us-east-2 --bucket copi-inbound-email \ + --prefix inbound/ --reply-domain reply.copi.science + +After provisioning +------------------ + 1. Add the printed MX (and, if newly verifying the domain, TXT) records. + 2. Attach the printed IAM policy to the instance role (copi-ec2-ses-role). + 3. Set ENABLE_INBOUND_EMAIL=true in the prod .env and recreate the worker: + docker compose -f docker-compose.prod.yml -f docker-compose.override.yml \ + up -d worker + 4. Send a test reply and watch: docker logs -f copi-python-worker-1 +""" + +import argparse +import json +import subprocess +import sys + +RULE_SET_NAME = "copi-inbound" +RULE_NAME = "copi-reply-to-s3" + + +def _print(status: str, layer: str, detail: str) -> None: + print(f" [{status:^4}] {layer}: {detail}") + + +def check_mx(reply_domain: str, region: str) -> bool: + """MX must point at SES inbound SMTP for the region.""" + expected = f"inbound-smtp.{region}.amazonaws.com" + try: + out = subprocess.run( + ["dig", "+short", "MX", reply_domain], + capture_output=True, text=True, timeout=10, + ).stdout.strip() + except (FileNotFoundError, subprocess.TimeoutExpired): + _print("SKIP", "DNS", f"`dig` unavailable — check manually that {reply_domain} " + f"has MX 10 {expected}") + return False + if expected in out: + _print("OK", "DNS", f"MX for {reply_domain} → {expected}") + return True + _print("FAIL", "DNS", f"no MX for {reply_domain} pointing at {expected} " + f"(got: {out or 'no MX record at all'})") + print(f" Add at the registrar: {reply_domain}. MX 10 {expected}.") + return False + + +def check_bucket(s3, bucket: str) -> bool: + try: + s3.head_bucket(Bucket=bucket) + _print("OK", "S3", f"bucket {bucket} exists and is reachable") + return True + except Exception as exc: + _print("FAIL", "S3", f"bucket {bucket}: {exc}") + return False + + +def check_identity(ses, reply_domain: str) -> bool: + try: + attrs = ses.get_identity_verification_attributes(Identities=[reply_domain]) + status = ( + attrs["VerificationAttributes"] + .get(reply_domain, {}) + .get("VerificationStatus", "NotFound") + ) + except Exception as exc: + _print("SKIP", "SES identity", f"cannot query ({exc})") + return False + if status == "Success": + _print("OK", "SES identity", f"{reply_domain} is verified") + return True + _print("FAIL", "SES identity", f"{reply_domain} verification status: {status}") + return False + + +def check_receipt_rule(ses, bucket: str, reply_domain: str) -> bool: + try: + active = ses.describe_active_receipt_rule_set() + except Exception as exc: + _print("SKIP", "SES receipt", f"cannot query receipt rule sets ({exc})") + return False + for rule in active.get("Rules", []): + recipients = rule.get("Recipients", []) + domain_match = not recipients or any( + r == reply_domain or r.endswith("@" + reply_domain) for r in recipients + ) + s3_actions = [a["S3Action"] for a in rule.get("Actions", []) if "S3Action" in a] + if rule.get("Enabled") and domain_match and any( + a["BucketName"] == bucket for a in s3_actions + ): + _print("OK", "SES receipt", + f"active rule '{rule['Name']}' delivers {reply_domain} → s3://{bucket}") + return True + name = (active.get("Metadata") or {}).get("Name") + _print("FAIL", "SES receipt", + f"active rule set {name or '(none)'} has no enabled rule delivering " + f"{reply_domain} to s3://{bucket}") + return False + + +def check_env_flag() -> bool: + """This checks the LOCAL environment only — the flag that matters is the + one in the prod .env consumed by the worker container.""" + import os + + val = os.environ.get("ENABLE_INBOUND_EMAIL", "") + if val.lower() in ("1", "true", "yes"): + _print("OK", "Env", "ENABLE_INBOUND_EMAIL is set here") + else: + _print("WARN", "Env", + "ENABLE_INBOUND_EMAIL not set in this shell — ensure it is " + "true in the prod .env (worker service) once AWS+DNS are ready") + return True + + +def instance_role_policy(bucket: str, prefix: str) -> dict: + """The statements copi-ec2-ses-role needs for the worker's polling loop + (read+delete under the inbound prefix, write for failed/ quarantine).""" + return { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "CopiInboundEmailList", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": f"arn:aws:s3:::{bucket}", + }, + { + "Sid": "CopiInboundEmailReadWrite", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:DeleteObject", "s3:PutObject"], + "Resource": [ + f"arn:aws:s3:::{bucket}/{prefix}*", + f"arn:aws:s3:::{bucket}/failed/*", + ], + }, + ], + } + + +def ses_bucket_policy(bucket: str, account_id: str, region: str) -> dict: + """Allow SES (this account's receipt rules only) to write into the bucket.""" + return { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowSESPuts", + "Effect": "Allow", + "Principal": {"Service": "ses.amazonaws.com"}, + "Action": "s3:PutObject", + "Resource": f"arn:aws:s3:::{bucket}/*", + "Condition": { + "StringEquals": {"AWS:SourceAccount": account_id}, + "ArnLike": { + "AWS:SourceArn": f"arn:aws:ses:{region}:{account_id}:receipt-rule-set/*" + }, + }, + } + ], + } + + +def provision(region: str, bucket: str, prefix: str, reply_domain: str) -> None: + import boto3 + + account_id = boto3.client("sts", region_name=region).get_caller_identity()["Account"] + s3 = boto3.client("s3", region_name=region) + ses = boto3.client("ses", region_name=region) + + # 1. Bucket (idempotent) + SES write policy + try: + s3.head_bucket(Bucket=bucket) + print(f"bucket {bucket} already exists") + except Exception: + kwargs = {"Bucket": bucket} + if region != "us-east-1": + kwargs["CreateBucketConfiguration"] = {"LocationConstraint": region} + s3.create_bucket(**kwargs) + s3.put_public_access_block( + Bucket=bucket, + PublicAccessBlockConfiguration={ + "BlockPublicAcls": True, "IgnorePublicAcls": True, + "BlockPublicPolicy": True, "RestrictPublicBuckets": True, + }, + ) + print(f"created bucket {bucket}") + s3.put_bucket_policy( + Bucket=bucket, Policy=json.dumps(ses_bucket_policy(bucket, account_id, region)) + ) + print("attached SES write policy to bucket") + + # 2. Domain identity for receiving (prints the TXT record if new) + attrs = ses.get_identity_verification_attributes(Identities=[reply_domain]) + status = ( + attrs["VerificationAttributes"].get(reply_domain, {}).get("VerificationStatus") + ) + if status != "Success": + token = ses.verify_domain_identity(Domain=reply_domain)["VerificationToken"] + print(f"requested domain verification for {reply_domain}; add DNS record:") + print(f' _amazonses.{reply_domain}. TXT "{token}"') + + # 3. Receipt rule set + rule (idempotent), then activate + try: + ses.create_receipt_rule_set(RuleSetName=RULE_SET_NAME) + print(f"created receipt rule set {RULE_SET_NAME}") + except ses.exceptions.AlreadyExistsException: + print(f"receipt rule set {RULE_SET_NAME} already exists") + rule = { + "Name": RULE_NAME, + "Enabled": True, + "Recipients": [reply_domain], + "Actions": [ + { + "S3Action": { + "BucketName": bucket, + "ObjectKeyPrefix": prefix, + } + } + ], + "ScanEnabled": True, + "TlsPolicy": "Optional", + } + try: + ses.create_receipt_rule(RuleSetName=RULE_SET_NAME, Rule=rule) + print(f"created receipt rule {RULE_NAME}") + except ses.exceptions.AlreadyExistsException: + ses.update_receipt_rule(RuleSetName=RULE_SET_NAME, Rule=rule) + print(f"updated receipt rule {RULE_NAME}") + active = ses.describe_active_receipt_rule_set().get("Metadata") or {} + if active.get("Name") != RULE_SET_NAME: + if active.get("Name"): + print(f"WARNING: replacing active rule set {active['Name']!r} — its rules " + f"stop matching. Merge them into {RULE_SET_NAME} first if needed.") + ses.set_active_receipt_rule_set(RuleSetName=RULE_SET_NAME) + print(f"activated receipt rule set {RULE_SET_NAME}") + + # 4. What cannot be done from here + print("\nRemaining manual steps:") + print(f" 1. Registrar DNS: {reply_domain}. MX 10 " + f"inbound-smtp.{region}.amazonaws.com.") + print(" 2. Attach this policy to the EC2 instance role (copi-ec2-ses-role):") + print(json.dumps(instance_role_policy(bucket, prefix), indent=4)) + print(" 3. Set ENABLE_INBOUND_EMAIL=true in the prod .env and recreate the worker.") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--check", action="store_true", default=False) + ap.add_argument("--provision", action="store_true", default=False) + ap.add_argument("--region", default="us-east-2") + ap.add_argument("--bucket", default="copi-inbound-email") + ap.add_argument("--prefix", default="inbound/") + ap.add_argument("--reply-domain", default="reply.copi.science") + args = ap.parse_args() + + if args.provision: + provision(args.region, args.bucket, args.prefix, args.reply_domain) + return 0 + + # Default: --check + import boto3 + + s3 = boto3.client("s3", region_name=args.region) + ses = boto3.client("ses", region_name=args.region) + print(f"Inbound email infrastructure check ({args.reply_domain} → " + f"s3://{args.bucket}/{args.prefix} in {args.region}):") + results = [ + check_mx(args.reply_domain, args.region), + check_identity(ses, args.reply_domain), + check_receipt_rule(ses, args.bucket, args.reply_domain), + check_bucket(s3, args.bucket), + check_env_flag(), + ] + if all(results): + print("All layers OK.") + return 0 + print("\nOne or more layers missing — run with --provision (admin creds) " + "and follow the printed manual steps.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/slack_test_teardown.py b/scripts/slack_test_teardown.py new file mode 100755 index 0000000..59b09a2 --- /dev/null +++ b/scripts/slack_test_teardown.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Archive t-prefixed test channels and delete *ProbeBot apps in the test workspace. + +Refuses to touch anything outside those two patterns. Run --dry-run first. + +NEVER drops the copi_slack_test database: _config_token() writes the rotated Slack +app-config credential pair into its app_settings table, and the refresh token it was +rotated from is single-use and already dead. Dropping that database loses Slack +app-configuration access permanently. + + SLACK_CONFIG_TOKEN=... SLACK_TEST_BOT_TOKEN_SU=... python scripts/slack_test_teardown.py --dry-run +""" +import argparse +import json +import os +import sys +import urllib.parse +import urllib.request + +API = "https://slack.com/api" +CHANNEL_PREFIX = "t-" +APP_NAME_SUFFIX = "ProbeBot" + + +def call(method, token, payload=None, form=False): + if form: + data = urllib.parse.urlencode(payload or {}).encode() + headers = {"Authorization": f"Bearer {token}"} + else: + data = json.dumps(payload or {}).encode() + headers = {"Authorization": f"Bearer {token}", + "Content-Type": "application/json; charset=utf-8"} + req = urllib.request.Request(f"{API}/{method}", data=data, headers=headers) + with urllib.request.urlopen(req) as r: + return json.load(r) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + + bot = os.environ.get("SLACK_TEST_BOT_TOKEN_SU") + if not bot: + sys.exit("SLACK_TEST_BOT_TOKEN_SU is required to list/archive channels") + + cursor, targets = "", [] + while True: + d = call("conversations.list", bot, { + "types": "public_channel,private_channel", "limit": 200, "cursor": cursor, + }, form=True) + if not d.get("ok"): + sys.exit(f"conversations.list failed: {d.get('error')}") + for c in d["channels"]: + if c["name"].startswith(CHANNEL_PREFIX) and not c.get("is_archived"): + targets.append(c) + cursor = (d.get("response_metadata") or {}).get("next_cursor") or "" + if not cursor: + break + + print(f"channels to archive ({len(targets)}):") + for c in targets: + print(f" #{c['name']} {c['id']}") + if not args.dry_run: + r = call("conversations.archive", bot, {"channel": c["id"]}, form=True) + if not r.get("ok"): + print(f" WARNING: {r.get('error')}") + + cfg = os.environ.get("SLACK_CONFIG_TOKEN") + if not cfg: + print("\nSLACK_CONFIG_TOKEN not set — skipping app deletion") + return + apps_file = os.environ.get("PROBE_APPS_JSON") + if not apps_file or not os.path.exists(apps_file): + print("\nPROBE_APPS_JSON not set — skipping app deletion " + "(there is no list-apps API for config tokens)") + return + apps = json.loads(open(apps_file).read()) + print(f"\napps to delete ({len(apps)}):") + for aid, a in apps.items(): + if not a["bot_name"].endswith(APP_NAME_SUFFIX): + print(f" SKIP {a['bot_name']} — does not match *{APP_NAME_SUFFIX}") + continue + print(f" {a['bot_name']} {a['app_id']}") + if not args.dry_run: + r = call("apps.manifest.delete", cfg, {"app_id": a["app_id"]}) + if not r.get("ok"): + print(f" WARNING: {r.get('error')}") + + +if __name__ == "__main__": + main() diff --git a/scripts/spike_private_channels.py b/scripts/spike_private_channels.py index ad2d734..42f2e5d 100644 --- a/scripts/spike_private_channels.py +++ b/scripts/spike_private_channels.py @@ -6,15 +6,15 @@ then archives the channel. Usage: - docker exec copi-python-opus-app-1 python3 scripts/spike_private_channels.py \\ + docker compose exec app python3 scripts/spike_private_channels.py \\ --bot-a su --bot-b wiseman # Optional: also invite a human user to verify the PI-invite path - docker exec copi-python-opus-app-1 python3 scripts/spike_private_channels.py \\ + docker compose exec app python3 scripts/spike_private_channels.py \\ --bot-a su --bot-b wiseman --pi-user-id U01234567 # Optional: also test the negative case (uninvited bot tries to post) - docker exec copi-python-opus-app-1 python3 scripts/spike_private_channels.py \\ + docker compose exec app python3 scripts/spike_private_channels.py \\ --bot-a su --bot-b wiseman --uninvited-bot lotz What it checks: diff --git a/scripts/wipe_slack.py b/scripts/wipe_slack.py index 856f456..357c378 100644 --- a/scripts/wipe_slack.py +++ b/scripts/wipe_slack.py @@ -14,19 +14,19 @@ Usage: # See what would be deleted (safe): - docker exec copi-python-opus-app-1 python3 scripts/wipe_slack.py \ + docker compose exec app python3 scripts/wipe_slack.py \ --workspace T0123ABCD --dry-run # Actually delete (asks for confirmation): - docker exec -it copi-python-opus-app-1 python3 scripts/wipe_slack.py \ + docker compose exec app python3 scripts/wipe_slack.py \ --workspace T0123ABCD # Non-interactive delete + reset memories: - docker exec copi-python-opus-app-1 python3 scripts/wipe_slack.py \ + docker compose exec app python3 scripts/wipe_slack.py \ --workspace T0123ABCD --yes --memory # Only reset working memories (no Slack access): - docker exec copi-python-opus-app-1 python3 scripts/wipe_slack.py --memory-only + docker compose exec app python3 scripts/wipe_slack.py --memory-only """ import argparse diff --git a/specs/cohort-system-v2.md b/specs/cohort-system-v2.md new file mode 100644 index 0000000..8fb215b --- /dev/null +++ b/specs/cohort-system-v2.md @@ -0,0 +1,1210 @@ +# Cohort System — Specification v2 + +**Status:** Implemented on branch `cohort-db-conversations` (not merged to `main`) +**Date:** 2026-07-30 +**Supersedes:** `specs/cohort-system.md` (v1, added in `c0514e5`) +**Audited implementation (v1):** `origin/cohort-agent-isolation` @ `b00b0e6` (1,058 lines) +**v2 implementation:** branch `cohort-db-conversations`, merging that branch onto +`main`'s DB-primary conversation store and then building this specification out in full +**Audit basis:** static review, unit + integration probes, live migration execution +against clones of the running `copi` database, and an adversarial pass against the v2 +implementation itself (which found two defects — see §5.3 and §9) + +--- + +## 0. Why there is a v2 + +v1 was written and then implemented on a branch cut from the same commit. During +implementation, several v1 decisions were deliberately reversed and several were +silently dropped. v1 was never amended, so `main` currently documents a design +that does not exist and promises behaviour that the code inverts. + +| v1 claim | Observed reality | v2 decision | +|---|---|---| +| Uncohorted agents interact with everyone (v1 §Agent Changes, §Backward Compatibility) | `allowed_sender_ids = set()` — uncohorted agents are **isolated**; enabling the flag with zero cohorts silences the whole roster | §5: default `open`, explicit opt-in to `isolated`, mandatory preflight | +| Min-heap turn selection + global semaphore of `concurrent_turns` (v1 §6) | Never implemented in any branch; replaced by a sequential **reactive-priority** scheduler | §10: reactive priority is the design of record; v1 §6/§7 retired | +| `turn_delay_seconds` becomes a per-agent cooldown (v1 §Configuration) | Still a global `asyncio.sleep`; selection ignores it entirely | §10.3: implement as eligibility filter | +| Gate applied inside Phase 2/3/5 (v1 §3–§5) | Applied at the `MessageLog` read boundary — **better**, one choke point | §6: keep, and complete the coverage | +| Migration `0023_add_cohorts.py` | Shipped as `0019_add_cohorts.py`, colliding with `main`'s `0019_agent_message_content.py` | §14: renumber + CI gate | +| Cohort detail page shows an audit log; delete blocked while members exist | Neither implemented | §12 | +| Cost framing: "LLM calls scale as O(n²)" (infographic) | False under the sequential scheduler — call *rate* is O(1) in roster size | §3: corrected cost model | +| "Cohorts are orthogonal to Slack channels" | True of subscriptions, false in effect: a PI-created private channel goes silent across cohorts | §7: channel-level exemption | +| Gate at the `MessageLog` read boundary covers everything | True for the in-memory log, but `main` added DB read paths that bypass `MessageLog` entirely (DM inbox, state rebuild, PI-facing web views) | §6.2: second inventory + normative ingest prohibition | +| `sender_agent_id is None` means "human" | `agent_messages.agent_id` is **nullable** on `main`; a NULL-agent bot row ingests as `sender_agent_id=None` and silently passes the gate as a human | §5.1: key the human bypass on `is_bot` | +| Grandfathered threads are an edge case | Every *resumed* run rebuilds all open threads with `allowed_sender_ids = None` — cohort-blind — because the rebuild runs in setup, before the first recompute | §8: reframed as the normal path | + +v1 was also written against the pre-`db-primary-conversations` engine. `main` has +since moved the durable conversation store into Postgres (PR #19, 56 commits after +the fork). §6.2, §6.3 and §8 are new in v2 and exist only because of that change. +Read this document alongside `specs/local-db-conversations.md`. + +Everything in the tables above was confirmed by execution or by reading `main`. +See Appendix A. + +--- + +## 1. Terminology — resolve the name collision first + +`main` already uses **cohort** for something else: date-bounded slices of one +long-running simulation, used by the public graph routes +(`CABO_COHORT_START`, `JUNE_POST_START`, the `cohort_posts` CTE in +`src/routers/public.py`, and `scripts/build_cabo_sankey.py`). Introducing a +`cohorts` table for agent grouping puts two unrelated meanings in one codebase. + +**Decision:** the agent-grouping concept keeps the name *cohort* (tables, models, +admin UI, this spec). The pre-existing graph concept is renamed to **run window** +in comments, local variables, and function parameters — it has no table, so the +rename is comment-and-parameter churn only: + +- `CABO_COHORT_START` → `CABO_WINDOW_START` +- `cohort_start` parameter → `window_start_bound` +- `cohort_posts` CTE → `window_posts` +- `build_cabo_sankey.py` docstring: "simulation cohort" → "run window" + +This is a cheap rename and it must land **before** the `cohorts` table, or every +future grep for "cohort" returns two concepts. + +**Done** on `cohort-db-conversations`: no test referenced any of these names, so the +rename was internal. The dangling pointers to `memory project_reunion_cohort_boundary` +and `project_graph_cohort_windows` — which resolve to nothing in the repo — were +replaced with references to the window constants themselves. `src/routers/agent_page.py`'s +stale "cross-cohort interaction inactivation" comment now says what it means and +cross-references §7. + +--- + +## 2. Scope + +**In scope.** A cohort is a named, admin-managed group of agents. Cohort +membership gates whether one agent will *act on* another agent's activity during +simulation. Agents may belong to any number of cohorts. Membership is editable +while a run is live and takes effect without a restart. Per-agent limits (thread +count, proposal caps, budgets) stay per-agent and are shared across cohorts. + +**Out of scope.** Agent-visible cohort identity; PI-managed cohorts; per-cohort +budgets; cohort-scoped history or separate Slack workspaces; time-bounded +memberships; any role in turn *scheduling* (see §10 — the scheduler and the gate +are independent features that v1 conflated). + +--- + +## 3. Corrected cost model + +v1's goals section and the infographic justify cohorts with an O(n²) LLM-cost +argument. That argument is invalid for the scheduler that exists, and stating it +invites the wrong design decisions. + +Under the sequential loop, per turn: + +| Phase | LLM calls | Scales with roster size? | +|---|---|---| +| 1 Channel discovery | 0 | — | +| 2 Scan & filter | **1** (batched over all new posts) | No — call count fixed; *prompt tokens* grow with post volume | +| 3 Activate threads | 0 | — | +| 4 Reply threads | ≤ `active_thread_threshold` | No — capped per agent | +| 5 New post | ≤ 1 | No | + +Turn rate is set by wall clock, not by roster size. So **adding agents does not +increase the LLM call rate at all.** What grows is (a) Phase-2 prompt tokens, +(b) contention for each agent's capped thread slots, and (c) each agent's +turn *interval* (more agents, same turns/hour, so each waits longer). + +What cohorts actually buy: + +1. **Fewer Phase-2 prompt tokens** — the scan prompt only carries cohort-mates' posts. +2. **Better thread-slot allocation** — an agent's `active_thread_threshold` slots + are not consumed by partners it will never productively engage. +3. **Fewer wasted Phase-5 tags** — no dangling tags toward agents that won't answer. + +All three are real. None is a call-count reduction. Write it this way in any +future document, and delete the O(n²) claim from `docs/cohort-infographic.html` +(currently on `coPI-podcast` only). + +--- + +## 4. Data model + +### 4.1 Tables + +```sql +CREATE TABLE cohorts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + description TEXT, + created_by UUID REFERENCES users(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT now() +); + +CREATE TABLE cohort_memberships ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + cohort_id UUID NOT NULL REFERENCES cohorts(id) ON DELETE CASCADE, + agent_id TEXT NOT NULL, + added_by UUID REFERENCES users(id) ON DELETE SET NULL, + added_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + UNIQUE (cohort_id, agent_id) +); +CREATE INDEX ix_cohort_memberships_cohort_id ON cohort_memberships (cohort_id); +CREATE INDEX ix_cohort_memberships_agent_id ON cohort_memberships (agent_id); + +-- New in v2 (v1 promised an audit log and never specified or built one). +CREATE TABLE cohort_audit_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + cohort_id UUID, -- no FK: survives cohort deletion + cohort_name TEXT NOT NULL, -- denormalised for post-delete readability + agent_id TEXT, -- NULL for cohort-level events + action TEXT NOT NULL, -- created|deleted|agent_added|agent_removed|isolation_enabled|isolation_disabled + actor_id UUID REFERENCES users(id) ON DELETE SET NULL, + actor_email TEXT, -- denormalised, survives user deletion + created_at TIMESTAMP WITH TIME ZONE DEFAULT now() +); +CREATE INDEX ix_cohort_audit_events_cohort_id ON cohort_audit_events (cohort_id); +CREATE INDEX ix_cohort_audit_events_created_at ON cohort_audit_events (created_at DESC); +``` + +`agent_id` carries no FK to `AgentRegistry` (agent rows may not exist when a +cohort is created); the application validates at join time — the shipped +implementation already does this correctly. + +The audit table is append-only and deliberately denormalised: a cohort deletion +cascades its memberships away, so the audit trail must not depend on either the +cohort row or the actor row surviving. + +### 4.2 Migration numbering — normative + +The v1 spec said `0023`. The implementation shipped `0019`, which **collides with +`main`'s `0019_agent_message_content.py`**. See §14 for the consequences, which +are severe. + +**Rule.** A migration's revision id is assigned at *merge* time, not at branch +time, and must be `max(existing revision ids) + 1`. As of `main` @ `b7edcbc` the +head is `0021`, so this feature ships as: + +``` +alembic/versions/0022_add_cohorts.py + revision = "0022" + down_revision = "0021" +``` + +If `main` advances before this lands, renumber again. A branch that has been +open long enough for `main` to add a migration **must** renumber before merge, +and CI must enforce it (§14.4). + +--- + +## 5. Gate semantics — normative + +This is the section v1 got wrong, and the one that determines whether the feature +is safe to switch on. + +### 5.1 The decision table + +For a viewing agent `V` and a log entry authored by `A`: + +| Condition | Entry visible to `V`? | +|---|---| +| Isolation disabled (`cohort_isolation_enabled = false`) | **Yes** — no filtering whatsoever | +| `A` is a human (PI, delegate, admin — **`entry.is_bot is False`**) | **Yes**, always | +| Entry is in a `collab_private` channel `V` belongs to | **Yes**, always (§7) | +| Entry belongs to a thread `V` already has open | **Yes** (§8) | +| `A` shares ≥ 1 cohort with `V` | **Yes** | +| `V` has no cohort memberships, and `cohort_default_policy = "open"` | **Yes** (default) | +| `V` has no cohort memberships, and `cohort_default_policy = "isolated"` | No | +| `A` has no cohort memberships, and `cohort_default_policy = "open"` | **Yes** | +| `A` is a bot that cannot be attributed to a cohort (unknown, or a NULL `agent_id`) | No — **fail closed**. See below | +| Otherwise | No | + +**Inbound fails closed, outbound fails open.** These pull in opposite directions +and the implementation reflects that deliberately: + +- **Inbound** (`_entry_allowed`): a bot message that cannot be attributed to a + cohort — an unknown `agent_id`, or a NULL one — is **filtered**. An agent that is + not in the running roster cannot reply anyway, so acting on its traffic spends + calls on a conversation that can never happen; and the NULL case is the hole + described below. v1's spec said "unknown sender — don't filter"; that was written + when the only sender was a live Slack bot. Under DB-primary, unattributable bot + rows are reachable, so the safe default inverts. +- **Outbound** (`_strip_disallowed_tags`): a mention naming a bot absent from + `_bot_name_to_id` is **left alone** and logged at WARNING. Here a false positive + mangles a human-readable message over what is usually roster lag, and the + receiving side filters it anyway. + +**Why `is_bot`, not `sender_agent_id is None`.** The shipped gate treats +`sender_agent_id is None` as the human signal. On the pre-`db-primary` engine those +coincided. They no longer do: `0019_agent_message_content` made +`agent_messages.agent_id` **nullable** ("NULL for human/PI messages"), and +`_poll_inbound_from_db` ingests rows as +`LogEntry(sender_agent_id=r.agent_id, is_bot=r.is_bot)`. Any bot-authored row +written with a NULL `agent_id` — by this engine, the web app, a backfill script, or +a future second process — therefore enters the log as `sender_agent_id=None` and +**passes the gate as a human**. `LogEntry.is_bot` is carried and persisted +independently, so keying on it closes the hole at zero cost. The model comment on +`AgentMessage.agent_id` ("every reader filters for a specific agent_id, so NULL +rows are naturally excluded") documents an invariant the cohort gate is the first +reader to break; update that comment when this lands. + +### 5.2 `cohort_default_policy` + +```python +cohort_isolation_enabled: bool = False +cohort_default_policy: Literal["open", "isolated"] = "open" +``` + +`"open"` reproduces v1's published contract: an agent in no cohort behaves +exactly as it does today. `"isolated"` reproduces the shipped implementation's +behaviour, for operators who want "cohort membership is mandatory to participate." + +Rationale for `"open"` as default: it makes `cohort_isolation_enabled = true` +safe to flip in isolation. Under the shipped behaviour, flipping the flag before +defining a single cohort silences every agent — confirmed by execution, and it is +also the state a fresh deployment is in. + +### 5.3 Mandatory preflight + +At engine start and on every membership resync, if `cohort_isolation_enabled` is +true: + +1. If `cohort_default_policy == "isolated"` **and no agent on the live roster has + any cohort membership** → log `ERROR`, treat isolation as **disabled** for this + tick, and surface a banner on `/admin/cohorts`. Never silently silence the roster. + + Count *live members*, not cohorts. "Zero cohorts defined" is the obvious case + (and the state a fresh deployment is in), but creating a cohort and never adding + anyone to it silences the roster just as completely — as does a cohort whose only + members are agents the engine is not running. Adversarial testing of the first + implementation of this rule found exactly that hole: it checked `cohort_count == 0` + and let an empty cohort through. +2. If `cohort_default_policy == "isolated"` and ≥ 1 agent is in no cohort → log + `WARNING` naming each isolated agent, and show the count in the admin UI. +3. If `session_factory` is unavailable → log `ERROR` and treat isolation as + disabled. The shipped code returns early and leaves the gate open with no + message, so the flag appears to work while doing nothing. + +### 5.4 Representation + +Keep the shipped `Agent.allowed_sender_ids: set[str] | None` — it is a better +primitive than v1's `cohort_ids` + `can_interact()` because it is computed once +per resync instead of per comparison. But make `None` mean exactly one thing: + +- `None` → gate disabled for this agent, no filtering. +- `set()` → gate active, no permitted senders. Only reachable under + `cohort_default_policy = "isolated"`. + +Add an assertion to the recompute path: under `policy = "open"`, an empty set is +a bug — emit `None` instead. + +--- + +## 6. Enforcement points + +The shipped implementation moved the gate from the phases to the `MessageLog` +read boundary. **Keep that** — it is one choke point instead of three, and all +three current call sites pass the gate correctly. But the coverage is incomplete: +of 11 read methods on `MessageLog`, 3 are gated. + +| Method | v2 requirement | +|---|---| +| `get_new_top_level_posts` | **Gated** (done) | +| `get_replies_to_agent_posts` | **Gated** (done) | +| `get_tags_for_agent` | **Gated** (done) | +| `has_new_reply_from_other` | **Must take the gate.** Currently ungated; feeds both `_owes_reply` (scheduler) and the Phase-4 reply decision. See §8 | +| `get_thread_history` | Ungated **by design** — once a thread is open, its full history is context. Document it | +| `get_thread_allowed_agents` | Ungated by design — thread participation, not cohort | +| `get_agent_top_level_posts` | Ungated by design — an agent's own posts | +| `get_thread_message_count` | Ungated by design — bookkeeping | +| `get_last_bot_sender_in_channel` | Ungated by design — anti-monologue check | +| `load_entry`, `latest_timestamp` | Ungated by design — bookkeeping | + +Any new read method must declare its classification in its docstring. Add a test +that fails when a new public `get_*`/`has_*` method appears without a +classification comment, so the inventory cannot silently rot. + +Extend the rule to **private** reads. `_rebuild_agent_state` iterates +`self.message_log._entries` directly (`simulation.py:3302+`), bypassing every gated +accessor. Any code that touches `_entries` must be listed here with a +classification, or the inventory is decorative. + +### 6.1 Stale `interesting_posts` + +v1 §5 required pruning banked `interesting_posts` whose author left the cohort. +Not implemented. Because the gate is a read-time filter, posts accumulated +*before* a membership change keep driving Phase 5 indefinitely. + +**Requirement.** On every membership resync, for each agent under an active gate, +drop entries from `agent.state.interesting_posts` whose author is a bot not in +`allowed_sender_ids`. Log the count at DEBUG. + +Under DB-primary this is not cosmetic: `_rebuild_agent_state` restores +`interesting_posts` from the rebuilt log with no gate applied (§8), so the first +post-rebuild resync is the only thing that clears cohort-illegal banked posts. + +### 6.2 DB read paths — second inventory + +`MessageLog` is still the in-memory funnel: it remains append-only +(`message_log.py:1`, "single source of truth for the simulation"), the DB is +mirrored through a `_persist_cb`, and all eight feed sites in the engine go through +`message_log.append`. **So the read-boundary gate is still the right choke point** +— that part of the shipped design survives the DB-primary rewrite intact. + +But `main` added read paths that never touch `MessageLog`, and each needs an +explicit classification or someone will "helpfully" gate the wrong one: + +| Path | Classification | +|---|---| +| `_poll_inbound_from_db` (`simulation.py:2209`) | **Ingestion — must never be gated.** See below | +| `_hydrate_thread_from_db` (`:3012`) | Ingestion — never gated | +| `_rebuild_state_from_db` (`:2901`) | Ingestion — never gated | +| `_rebuild_agent_state` (`:3302`) | **Gate-blind state construction** — see §8 | +| `_poll_pi_dms_from_db` (`:2452`) → `PIHandler.handle_dm` | Never gated: humans only, and it bypasses `MessageLog` entirely | +| `src/routers/admin.py` reads of `AgentMessage` | **Never gated.** Admin-facing display | +| `src/routers/agent_page.py` conversations feed + thread-expand reads of `AgentMessage` | **Gated, as of 2026-08-05** — see amendment below. Every other `agent_page.py` read (dashboard, proposals, profile) stays ungated | + +**Normative: never filter at ingestion.** `MessageLog` is shared by every agent in +the process. `_poll_inbound_from_db` pulls rows for the whole +`simulation_run_id`, so applying `allowed_sender_ids` there would filter the log +for *all* agents according to one agent's cohort — silently corrupting the shared +store and, because ingestion advances `_pi_inbox_cursor`, dropping those rows +permanently. The gate belongs at the per-agent read, never at the write or the +ingest. This is the most tempting wrong move available once the conversation store +is a queryable database, so state it in the code as well as here. + +Corollary for the same reason: do **not** push the gate into SQL as a +`JOIN cohort_memberships` on the ingest query. A per-agent SQL gate would only be +correct in a future one-engine-per-cohort topology (§6.4), which is out of scope. + +**Normative, as written 2026-07-30: the gate is not access control.** It decides +what an agent *acts on*. It must never influence what a human sees. The PI +thread views, the admin discussion views, exports, and the public graph routes +read `AgentMessage` directly and must stay ungated. If cohort isolation ever +changes what a PI can read, that is a bug, not a feature. + +That held when written, against the flat-list conversations page of the time. It +no longer holds for one surface, deliberately. + +**Amendment, 2026-08-05: the PI conversations feed and thread endpoint are now +gated as read authorization, not just agent behaviour.** `GET +/agent/{agent_id}/conversations` and `GET +/agent/{agent_id}/thread/{message_ts}` (`src/routers/agent_page.py`) had **no** +content filter beyond channel name, so every PI's page showed every other lab's +bot traffic in `#general` and in any channel their own bot had posted in — the +deployed topology (56 agents, `cohort_isolation_enabled=True`, +`cohort_default_policy="isolated"`) already stopped a spoke bot from *acting +on* another spoke's posts, but the page showed it to the PI anyway. That is the +access-control leak this rule was meant to rule out, and the fix intentionally +narrows the rule rather than proving the leak was fine. + +`src/services/conversation_feed.py` now computes the same gate the engine +computes (`resolve_agent_gate`, via `compute_gates`) and renders it as a SQL +predicate (`gate_clause`, `own_or_gated`) applied to both routes before +`LIMIT`. This is a deliberate, narrow exception to the rule above — **do not +generalise it**: the admin discussion views, exports, and the public graph +routes are unaffected and must stay ungated, and this paragraph is the +authoritative list of what changed. A maintainer who finds the gated feed query +and concludes it contradicts this spec should update this paragraph, not +delete the filter — see `src/models/cohort.py`'s module docstring, updated to +match. + +### 6.3 Cursor semantics — filtering is forward-only + +`last_seen_cursor` is advanced to `time.time()` unconditionally when an agent takes +a turn (`simulation.py:648`), and the rebuild sets it to the latest message time +(`:3479`). The gate filters at read time *behind* that cursor. + +Consequence: **messages suppressed by the gate are suppressed permanently.** Adding +an agent to a cohort later does not reveal the backlog it missed — the cursor has +already moved past it. + +**Decision: accept this, and document it.** Replaying a backlog on membership +change would dump an arbitrary volume of stale posts into one Phase-2 prompt, which +is the opposite of the feature's purpose. Membership changes are forward-only. The +admin UI must say so next to the add-agent control, because the natural expectation +is the opposite. + +One interaction to preserve: `_rewind_cursors_for_private_channels` +(`:1494`, `:1554`) deliberately rewinds `last_seen_cursor` so agents re-scan +private-channel handovers the rebuild overshot. Before §7's exemption, the gate +discarded exactly those messages — the rewind ran and bought nothing. §7 fixes it; +add a test that pins the pair together. + +### 6.3.1 Membership writes must be atomic — normative + +The gate reads the whole `cohort_memberships` table on each recompute. If a writer +commits a wipe **separately** from the re-insert, any reader landing in the gap sees +an empty topology, the §5.3 preflight fires, and the gate goes **fully open** for that +tick. + +Measured with three real processes against one Postgres, 20 agents, ~600 topology +rewrites and ~5,800 recomputes: + +| writer | preflight refusals seen mid-churn | ticks with every gate open | +|---|---|---| +| single transaction (as shipped) | **0** of ~4,500 | **0** | +| wipe committed separately | 1,501 and 1,549 | ~57% of ticks | + +The shipped `/admin/cohorts/topology` route is safe: it stages every add and delete +and commits once. **Any future writer — a bulk CLI importer, a migration backfill, a +seeding script — must do the same.** Truncate-then-insert across two transactions +silently un-gates the roster about half the time under `policy="isolated"`, and +nothing in a code review would show it. Pinned by +`test_matrix_save_writes_memberships_atomically`. + +Note the failure is fail-*open*, never fail-closed: a transient empty read can only +make agents unrestricted, never silence them. That is the right direction, and it is +also why the symptom is easy to miss — nothing breaks, the gate just stops applying. + +### 6.4 Out of scope, but now newly possible + +With the DB as the durable store, `Cohort_approaches.txt`'s rejected Approach C +(one engine process per cohort) is viable in a way it was not when Slack was the +store: each process could load only its members and ingest only relevant rows, +making the gate structural instead of a filter. It also reopens the objection that +killed it — agents in overlapping cohorts posting from one bot token concurrently. +Not proposed here. Recorded so the option is not rediscovered from scratch. + +--- + +## 7. Private channels and PI overrides + +**Confirmed defect.** The reopen flow lets a PI explicitly pair two agents: it +creates a `collab_private` channel and adds it to both bots' +`subscribed_channels`. Phase 2 reads that channel through the gated call, so if +the two agents are in different cohorts the channel goes silent — the PI's own +handover message survives (human sender), the partner the PI chose does not. + +An admin-level grouping must not veto an explicit human pairing. + +**Requirement.** The gate does not apply to entries in channels whose visibility +is `collab_private`. Implement as an entry-level bypass evaluated *before* the +sender check, so it cannot be reordered away: + +```python +def _entry_allowed(entry: LogEntry, allowed_sender_ids: set[str] | None) -> bool: + if allowed_sender_ids is None: + return True # gate disabled for this agent + if not entry.is_bot: + return True # human — §5.1 + if entry.visibility == VISIBILITY_COLLAB_PRIVATE: + return True # PI-created pairing outranks the gate + return entry.sender_agent_id in allowed_sender_ids +``` + +Note this reads `LogEntry.visibility`, **not** the engine's in-memory +`_channel_visibility` map. `visibility` is already carried on every `LogEntry` +(`message_log.py`, added for the G2 memory-synthesis filter) and already persisted +on `AgentMessage.visibility`, so it survives restart and arrives correctly on rows +ingested by `_poll_inbound_from_db` from another process. Threading the engine's +map into `MessageLog` would work today and drift the moment a channel's visibility +is changed by the web app between resyncs. Use the persisted field. + +Rename the helper from `_sender_allowed` to `_entry_allowed`: it is no longer a +function of the sender alone, and the old name invites someone to re-narrow it. + +Corollary: the admin UI must state that private-channel collaborations bypass +cohort isolation, or admins will report it as a bug. + +--- + +## 8. Grandfathered threads + +v1 said threads orphaned by a membership change "are allowed to conclude +naturally." That is the right call — killing a live conversation mid-flight wastes +the calls already spent. But it interacts badly with §10's scheduler, and under +DB-primary it is not the edge case v1 assumed. + +**Every resumed run starts cohort-blind.** Setup runs +`_rebuild_state_from_db()` → `_rebuild_state_from_slack()` → `_rebuild_agent_state()` +(`simulation.py:396-398`), and only then enters the main loop, whose first act is +`_sync_roster_from_db()` (`:446`) — the sole caller of +`_recompute_allowed_sender_ids()`. So during the entire rebuild every agent's +`allowed_sender_ids` is still `None`, and `_rebuild_agent_state` reconstructs +`active_threads` by walking `message_log._entries` directly, with no gate at any +point. A run resumed after any restart therefore comes up with **every** previously +open partnership intact, regardless of cohort topology. Grandfathering is the normal +path, not an exception, and the count is worth logging on every start. + +This also means the gate's first effect on a resumed run is at the first resync, +several seconds in — never during rebuild. Do not "fix" that by gating the rebuild +reads: the rebuild populates the shared log (§6.2) and must stay complete. + +**Confirmed defect.** `has_new_reply_from_other` is ungated, so a reply from a +non-cohort agent still marks the recipient as owing a reply, and the reactive tier +selects that agent **ahead of every gate-compliant agent**. The gate says "ignore +this sender" while the scheduler says "answer them first." + +**Requirements.** + +1. `ThreadState` gains `grandfathered: bool = False`. On membership resync, any + active thread whose `other_agent_id` is no longer permitted is marked + `grandfathered = True` (once; never unset except by re-permission). Because of + the rebuild ordering above, the **first** resync after start is where a resumed + run's cross-cohort threads get marked — so the recompute must run before any + turn is taken. It already does (`_last_roster_poll` starts at `0.0`, and the + resync precedes `_select_agent()` in the loop body), but pin it with a test: + a resumed run must never take a turn with `allowed_sender_ids is None` while + `cohort_isolation_enabled` is true. +2. Grandfathered threads **do** get Phase-4 replies — they conclude normally, up + to the existing 12-message cap. +3. Grandfathered threads are **excluded from the reactive-priority tier** + (§10.2). They drain at proactive cadence. Rationale: they are the one class of + work the operator has signalled they don't want, so they must not outrank + everything else. +4. `has_new_reply_from_other` takes `allowed_sender_ids` and applies the §5.1 + table, with the open-thread row implemented as: the caller passes `None` when + the thread is already open and non-grandfathered. +5. Log every grandfathering event at INFO with agent, partner, thread id. + +--- + +## 9. Outbound tag hygiene + +The shipped `_strip_disallowed_tags` is wired into Phase 5 only, and has two +sharp edges, both confirmed: + +- With an empty `allowed_sender_ids`, it de-`@`s **every** bot mention, mangling + message text ("Great point @WisemanBot" → "Great point WisemanBot"). Under §5.2 + the empty set becomes rare, but it must still be handled. +- A bot name absent from `_bot_name_to_id` passes untouched, while a known + non-mate is stripped. The gate is inconsistent between known and unknown + targets. + +**Requirements.** + +1. Apply the strip on **every** outbound path — Phase 4 replies and + `_post_message`, not just Phase 5. Placing it in `_post_message` covers all + callers and is the only placement that cannot be bypassed by a new call site. +2. Unknown bot names: leave the tag, log at WARNING (roster lag is an operational + problem, not a policy decision). Match §5.1's fail-open row. +3. Replace the `@Name` → `Name` rewrite with removal of the whole mention, so the + sentence still reads. Keeping the bare name produces text that looks like an + addressed message but isn't. + + Do **not** globally normalise whitespace afterwards: agents put code blocks and + nested bullet lists in messages, and stripping line-leading whitespace mangles + them. Swallow the run of spaces/tabs immediately *before* the mention as part of + the match, then tidy only interior double-spaces (after a non-space) and + end-of-line space. Adversarial testing caught a first implementation that + flattened indentation across the whole message. + + Require the `@` to start a token (a lookbehind rejecting `\w`, `.`, `/`, `-`, + `@`). Since the strip now runs on every outbound message, without this an email + address or a URL path ending in a bot name gets mangled. +4. Count strips per agent per run and expose the total in the admin UI. A high + strip rate means the cohort topology disagrees with what the agents want to do + — that is a signal worth seeing. + +### 9.1 Slack-off mode + +With `NullTransport` (`slack_enabled` false), `is_connected` is `False`, every +Slack poller no-ops, and **all** inbound arrives through the DB inbox. The gate's +correctness in that mode therefore rests entirely on read-side filtering — there is +no second path that would incidentally catch a miss. Two constraints follow: + +- Tags remain plain `@BotName` text in `content`, so §9's stripping and + `get_tags_for_agent` work unchanged. Nothing in the gate may key on Slack + identifiers. +- Canonical ids are locally minted (`mint_ts`), so `slack_ts`, `slack_channel_id` + and `slack_thread_ts` are `None` on DB-origin entries. The gate must key only on + `sender_agent_id`, `is_bot`, and `visibility`. A gate that keys on `slack_ts` + would pass everything in Slack-off mode and nothing would look wrong. + +Run the whole §15 suite with `slack_enabled` both true and false. A gate that only +works with Slack on is a gate that fails exactly in the configuration the project +is moving toward. + +--- + +## 10. Scheduler — supersedes v1 §6 and §7 + +v1 §6 (min-heap + `concurrent_turns` global semaphore) and §7 (concurrent +pair-initiation guard) are **retired**. `concurrent_turns` exists in no branch; +`_build_heap` and `_run_concurrent_turns` were never written. The implementation +instead shipped a sequential two-tier scheduler, which is the design of record. + +Note for reviewers: cohorts and the scheduler are **independent**. They were +specified and shipped together, which is why neither can currently be evaluated +on its own. Land them as two changes. + +### 10.1 Why concurrency was dropped + +The observed problem was not throughput, it was that 1:1 threads stalled waiting +for staleness-weighted random re-selection. Concurrency does not fix that; +priority does. Concurrency also brings real costs the sequential loop avoids: +duplicate pair initiation (v1 §7 existed only to patch this), interleaved Slack +posting from one bot token, and non-deterministic `AgentState` mutation. Dropping +it was correct. It should have been written down. + +### 10.2 Reactive-priority selection + +Two tiers, sequential, one agent per turn: + +1. **Reactive** — agents that owe a thread reply, oldest-waiting first, excluding + `_last_llm_caller` (so an A→B→A baton alternates without a wasted skip tick) + and excluding grandfathered threads (§8). Bounded by + `max_consecutive_reactive_turns`. +2. **Proactive** — the existing staleness-weighted random selection, with the + Phase-5-skip penalty (`weight /= 2^(skips-2)` once `skips >= 3`). + +### 10.3 Fairness — the valve default must change + +`max_consecutive_reactive_turns = 8` was measured: with two agents in a live +thread and three idle, **24 of 27 selections went to the pair**. That is 8:1, and +v1's stated goal was the opposite ("ensure fair turn distribution across all +agents"). + +**Requirements.** + +1. Default `max_consecutive_reactive_turns = 3`, matching `active_thread_threshold` + so the two levers stay in proportion. Document the 8:1 → 3:1 change. +2. Add the two eligibility filters v1 specified and the implementation omitted — + the candidate pool is currently `_agent_within_budget` only: + - `(now - a.state.last_selected) >= turn_delay_seconds` (per-agent cooldown) + - agent is not administratively paused, if/when a pause flag lands on `main` + (`is_paused` exists only on `coPI-podcast` today — do not reference it until + it does) +3. Once the cooldown is enforced at selection time, remove the global + `asyncio.sleep(turn_delay_seconds)` from the main loop. Not before: removing it + first raises the turn rate with no throttle. +4. Emit a per-100-turn ratio of reactive:proactive selections to the run log, so + starvation is observable rather than inferred. + +--- + +## 11. Configuration + +```python +# Cohort interaction gate +cohort_isolation_enabled: bool = False +cohort_default_policy: Literal["open", "isolated"] = "open" + +# Scheduler (independent of cohorts) +max_consecutive_reactive_turns: int = 3 +``` + +Membership resync rides the existing `ROSTER_POLL_INTERVAL = 30.0` tick. v1 +specified a separate 60 s timer; reusing the roster tick is simpler, already +implemented, and correct — `_last_roster_poll` initialises to `0.0`, so the first +recompute happens before the first turn (verified). Do not add a second timer. + +**Membership is live; the settings are not.** `get_settings()` is `@lru_cache`d, so +`cohort_isolation_enabled`, `cohort_default_policy` and +`max_consecutive_reactive_turns` are read once per process. Editing the topology in +the admin UI takes effect within ~30 s with no restart; **changing the flag or the +policy requires restarting `agent-run`.** Verified by execution: setting the env var +in a live process leaves `get_settings()` returning the cached value. Say so in the +admin banner, or an operator will flip the flag and conclude the feature is broken. + +Retired settings: `concurrent_turns` (never existed), `COHORT_RESYNC_INTERVAL` +(subsumed by the roster tick). + +--- + +## 12. Admin interface + +Routes as shipped, all under `get_admin_user`: + +| Method | Path | Notes | +|---|---|---| +| GET | `/admin/cohorts` | list + inline create form | +| POST | `/admin/cohorts/create` | name validated `^[a-z0-9-]{1,48}$`, uniqueness checked | +| GET | `/admin/cohorts/{id}` | members, add-agent picker, agent→cohorts map, **audit log** | +| POST | `/admin/cohorts/{id}/delete` | **must refuse while members exist** | +| POST | `/admin/cohorts/{id}/add-agent` | validates agent exists in `AgentRegistry`; rejects duplicates | +| POST | `/admin/cohorts/{id}/remove-agent` | — | + +The shipped implementation already does name validation, the unknown-agent guard, +and the duplicate guard correctly, and the agent→cohorts map exists. Two gaps and +three additions: + +**Gaps to close.** + +1. **Delete guard.** The route currently deletes unconditionally and cascades; + only a JS `confirm()` mentions the member count. Enforce server-side: if + `memberships` is non-empty, redirect back with + `?error=Remove+all+members+first`. Disable the button in the template too — but + the server check is the one that counts. +2. **Audit log.** Render `cohort_audit_events` for the cohort, newest first, on + the detail page. Write an event from every mutating route (§4.1). + +**Additions required by §5.** + +3. Banner on `/admin/cohorts` when `cohort_isolation_enabled` is true, stating the + active `cohort_default_policy` and the number of agents currently isolated. +4. Red banner when the §5.3 preflight has forced isolation off, with the reason. +5. A note that `collab_private` channels bypass the gate (§7). + +--- + +## 13. Observability + +Nothing in the shipped implementation makes the gate's effect visible, so an +operator cannot tell whether it is working, over-filtering, or silently disabled. +Minimum: + +- Per resync, at INFO: cohort count, membership count, number of agents with an + active gate, number isolated. +- Per turn, at DEBUG: entries filtered by the gate, per agent. +- Per run, in the admin UI: total gate-filtered entries, total tags stripped, + reactive:proactive selection ratio, grandfathered-thread count. +- Every grandfathering event and every preflight override at INFO/ERROR (§5.3, §8). + +### 13.1 Run-topology provenance + +Cohort memberships are global; conversations are scoped to a `simulation_run_id` +(`_poll_inbound_from_db` filters on it). So an admin can reshape the topology +mid-run and nothing records that it happened, which makes a completed run's output +un-attributable to the configuration that produced it. For a research system that +is the more expensive failure than any of the bugs in §5–§9. + +**Requirement.** At run start, and on every membership change during a run, write a +`cohort_audit_events` row carrying the full topology snapshot (cohort names → +member `agent_id`s) plus the active `cohort_isolation_enabled` / +`cohort_default_policy` values, tagged with `simulation_run_id`. One row per +change, not per tick. This is the record that lets someone later ask "which cohort +configuration produced these proposals?" and get an answer. + +--- + +## 14. Migration and deployment — the collision + +This section is the reason v2 exists at all. It was verified by running real +migrations against clones of the live database. + +### 14.1 The collision + +`0019_add_cohorts.py` (`revision = "0019"`, `down_revision = "0018"`) collides +with `main`'s `0019_agent_message_content.py` (identical revision and +down_revision). Git merges the two branches **cleanly** — zero conflicts, and the +full test suite passes (398 tests) — so nothing in code review or CI flags it. +Alembic emits only `UserWarning: Revision 0019 is present more than once`. + +The live `copi` database is at revision **`0018`** — the exact fork point. Both +`0019`s claim to be the next migration. + +### 14.2 What happens, by command + +| Command | Result | DB after | +|---|---|---| +| `alembic upgrade head` (the command in `README.md:40`) | `FAILED: Multiple head revisions are present` | unchanged — fails closed | +| `alembic upgrade heads` (what the error message suggests) | `FAILED: Requested revision 0021 overlaps with other requested revisions 0019` | unchanged — fails closed | +| **`alembic upgrade 0021`** (the natural next attempt) | **Succeeds, exit 0** | stamped `0021`, **one of the two `0019` migrations silently never ran** | +| `alembic current` afterwards | prints `0021 (head)` | reports healthy | +| `alembic heads` | prints `0019` and `0021 (head)` — two heads | — | +| `alembic downgrade 0018` (incident rollback) | **Crashes**: `UndefinedObjectError: index "ix_cohort_memberships_agent_id" does not exist` | unchanged — Postgres transactional DDL rolls the whole chain back | +| `alembic upgrade 0022` (any future migration) | `FAILED: Multiple head revisions are present` | unchanged, forever | + +### 14.3 The damage + +Alembic loads version files in `sorted(filename)` order and the **last** duplicate +wins the revision map. With the two files as they exist, +`0019_agent_message_content.py` sorts after `0019_add_cohorts.py`, so: + +- `alembic upgrade 0021` applies `agent_message_content`, `pi_dm_messages`, and the + inbox indexes, and **never creates `cohorts` / `cohort_memberships`.** The + database reports `0021 (head)`. The cohort feature is silently absent; the admin + UI 500s on first use. +- Rename either file — or add a third `0019_*` — and the winner flips. Verified by + execution: with `add_cohorts` sorting last, the identical command on the + identical starting database produced `cohorts` present and + **`agent_messages.content` missing**, still stamped `0021`. Runtime result: + `asyncpg.exceptions.UndefinedColumnError: column agent_messages.content does not + exist` — the entire DB-primary conversation store is broken while Alembic + reports the database fully migrated. + +So the outcome is deterministic given the filenames, but which schema change is +silently dropped depends on filenames alone, and neither the migration output nor +`alembic current` reveals that anything was skipped. + +**And the box is then wedged permanently.** Two heads means every future +`alembic upgrade head` fails; `downgrade` crashes in the wrong `0019`'s +`downgrade()` and rolls back. There is no forward and no backward path without +hand-editing revision ids. `alembic current` says `0021 (head)` throughout. + +Blast radius on a production database with live data: no destructive DDL runs +(every failure path rolled back atomically — `pi_dm_messages` and its rows +survived the failed downgrade). The damage is a **silently incomplete schema plus +a permanently unmigratable database**, not data loss. That is still an incident, +and the "reports healthy" property makes it one that gets discovered late. + +### 14.4 Required fixes + +1. **Renumber** to `0022_add_cohorts.py` / `revision = "0022"` / + `down_revision = "0021"`. +2. **Preflight gate in CI** — this is the durable fix; renumbering one file only + fixes one file. Add to `scripts/ci.sh`: + + ```bash + # Exactly one Alembic head, and no duplicate revision ids. + heads=$(alembic heads 2>/dev/null | grep -c .) + if [ "$heads" -ne 1 ]; then + echo "FAIL: expected 1 alembic head, found $heads"; alembic heads; exit 1 + fi + dupes=$(grep -h '^revision' alembic/versions/*.py | sort | uniq -d) + if [ -n "$dupes" ]; then + echo "FAIL: duplicate alembic revision ids:"; echo "$dupes"; exit 1 + fi + ``` + + `alembic check` is **not** sufficient — it reported `Target database is not up + to date`, which is the wrong diagnosis, and it needs a live database. +3. **Idempotent downgrades.** Both `0019`s crash on a partially applied schema. + Use `op.drop_index(..., if_exists=True)` / `DROP TABLE IF EXISTS` so a rollback + cannot wedge on an object that was never created. +4. **Deploy runbook.** Before any migration: record `alembic current` and + `alembic heads`; abort if `heads` returns more than one line. After: assert the + expected tables/columns exist, not just that `alembic current` advanced. Update + `README.md:40`, which currently documents the failing command. + +### 14.5 Separately: this box is three migrations behind its code + +The live `copi` database is at `0018`. `main`'s code expects `0021`. +`agent_messages` is missing `content`, `slack_ts`, `slack_channel_id`, and +`slack_thread_ts` — every column `0019_agent_message_content` adds. The +DB-primary conversation store on `main` cannot work against this schema. Run +`alembic upgrade head` on the **clean** `main` tree (single head, succeeds) before +any cohort work lands, and verify the four columns exist afterwards. + +--- + +## 14.6 Measured scale headroom + +Gate recompute against a real Postgres with 20 agents (the recompute runs every 30 s): + +| cohorts | memberships | p50 | p99 | +|---|---|---|---| +| 4 | 20 | 2.3 ms | 8.3 ms | +| 20 | 100 | 3.1 ms | 11.9 ms | +| 100 | 500 | 4.9 ms | 99.8 ms | +| 400 | 2000 | 11.0 ms | 109.3 ms | + +Free at any plausible roster size, and still affordable three orders of magnitude out. +No indexing or caching work is warranted; do not add a cache and reintroduce staleness +to solve a 3 ms problem. + +**Engine-level figure.** The table above times the gate computation. The full +`_recompute_allowed_sender_ids()` — session checkout, three queries, `compute_gates`, +then `_apply_cohort_gate_to_state` over every agent (grandfathering plus +`interesting_posts` pruning) — measures **p50 61 ms** at 20 agents / 100 cohorts / 500 +memberships, in-container against a networked Postgres. That is the number that matters +operationally, and it is the one pinned by +`test_gate_is_correct_and_affordable_at_20_agents` with a 500 ms bound (~8x headroom). +The bound is deliberately loose: it exists to catch an order-of-magnitude regression, +not to police jitter on a shared test box. That test also asserts the gates are +non-empty, self-inclusive and symmetric, because a recompute that returned empty gates +instantly would satisfy a timing bound alone — and an empty gate silences an agent. + +--- + +## 15. Test plan + +The shipped 20 tests pass and cover the filter, the recompute, `_owes_reply`, and +the reactive tier honestly. They are also the mechanism by which the inverted +semantics became load-bearing: `test_enabled_computes_cohort_mates` asserts +`allowed_sender_ids == set()` for an uncohorted agent, locking in the behaviour +v1 forbade. Rewrite that assertion against §5.1. + +Relocate to `tests/unit/test_cohort_isolation.py` — `main` moved to a four-tier +layout (`tests/unit`, `characterization`, `contract`, `integration`) in PR #17 and +the branch predates it, so the tests currently sit outside the CI gate. + +Required new coverage, one test per normative claim: + +**Gate semantics (§5)** +- `policy="open"` + isolation on + zero cohorts → every agent's gate is `None`; a + post from any agent is visible. +- `policy="open"` + partial cohorting → uncohorted agent sees the cohorted roster. +- `policy="isolated"` + uncohorted agent → sees only humans. +- `policy="isolated"` + zero cohorts → preflight forces isolation off, logs ERROR. +- No `session_factory` + isolation on → forced off, logs ERROR. +- Human sender always passes, under both policies. +- Unknown sender passes, logs WARNING. + +**Enforcement (§6)** +- Each gated method filters; each intentionally ungated method is asserted + ungated, so a future change is deliberate. +- New-method guard: every public `get_*`/`has_*` on `MessageLog` carries a + classification comment. +- Stale `interesting_posts` are pruned on resync. + +**Private channels (§7)** +- Two agents in different cohorts, isolation on, `collab_private` channel → both + the PI message *and* the partner's message are visible. + +**Grandfathered threads (§8)** +- Membership removal marks an open thread grandfathered; Phase 4 still replies. +- A grandfathered thread does **not** win the reactive tier. +- `has_new_reply_from_other` respects the gate for non-open threads. + +**Tag hygiene (§9)** +- Phase 4 and `_post_message` strip cross-cohort tags. +- Unknown bot name survives; WARNING logged. +- Stripped text reads cleanly (no bare dangling name). + +**Scheduler (§10)** +- With `max_consecutive_reactive_turns = 3`, a live pair takes ≤ 3 of every 4 + turns (the current default gives 24/27). +- An agent within `turn_delay_seconds` of its last turn is not selected. + +**DB-primary paths (§6.2, §6.3, §8)** +- A bot-authored row with `agent_id = NULL` ingested via `_poll_inbound_from_db` + does **not** pass the gate (keys on `is_bot`). +- `_poll_inbound_from_db` ingests every row for the run regardless of any agent's + cohort — assert the shared log is complete while a gated agent's read is filtered. +- A resumed run reconstructs a cross-cohort thread, and the first resync marks it + grandfathered; no turn is taken with `allowed_sender_ids is None` while isolation + is enabled. +- PI-facing reads (`agent_page`, `admin`) return unfiltered history for a + cross-cohort thread while isolation is on. +- PI DM handling is unaffected by any cohort configuration. +- Private-channel cursor rewind + §7 exemption together: the rewound agent actually + sees its partner's private-channel messages. +- Whole suite green with `slack_enabled` true **and** false (§9.1). + +**Real API (§15.1)** — opt-in, `real_llm` marker, skipped without a key +- Two real Phase 2 calls with the same profile and log, differing only in the gate: + ungated the model **selects** the excluded partner's post; gated it cannot. Assert + both legs — the ungated control is what makes the gated leg mean anything. +- The outbound strip removes a cross-cohort mention from real model prose. +- A real scan response parses, and can only name surviving post ids. + +**Migration (§14)** +- `alembic heads` returns exactly one line. +- No duplicate revision ids across `alembic/versions/`. +- Round-trip `upgrade` → `downgrade` → `upgrade` on a scratch database leaves the + expected schema, and each downgrade is idempotent. + +--- + +### 15.1 Real-API tests — the vacuity trap + +`tests/integration/test_cohort_real_llm.py` spends real tokens and is skipped unless +`ANTHROPIC_API_KEY` is set. It exists for one claim a fake cannot check: that the gate +changes the model's **decision**, not merely its prompt. + +The first version of it passed while proving nothing, and only capturing the raw model +output revealed why. With no profile loaded on the agent, the scan selected **no posts +in either leg** — so the gated assertion ("the model did not select the excluded post") +held for the wrong reason, and the ungated "control" demonstrated no difference at all. + +The fix is structural, and any future test here needs the same shape: give the agent a +profile that makes the *excluded* post directly relevant and the *surviving* post +clearly irrelevant, then assert on **both** legs. Measured: + +| | posts in prompt | model selected | +|---|---|---| +| gate off | both | `["1000.0002"]` — with reasoning citing the match | +| gate on | one | `[]` — considered only the survivor, rejected as out of scope | + +A real-API test whose passing condition is an *absence* (no leak, no mention, no +selection) is vacuous unless a paired positive leg shows the thing would otherwise be +present. Do not add one without it. + +Model IDs are read from settings (`llm_agent_model_sonnet`), not hardcoded, so the +suite follows the configured model. Both configured ids — `claude-opus-4-6` and +`claude-sonnet-4-6` — were verified live; pricing at the time of writing was +$5/$25 and $3/$15 per MTok respectively. + +--- + +### 15.2 Coverage as shipped + +The plan that closed the remaining gaps is `.notes/cohort-thorough-test-plan.md`. Its +organising principle is the failure mode above, generalised into two rules that every +new cohort test must follow: + +- **Rule A** — a test whose passing condition is an absence needs a positive control in + the same test. If the permitted leg does not fire, the result is *inconclusive*, not a + pass. Assertion messages say `INCONCLUSIVE` where that distinction matters. +- **Rule B** — let the system produce the state you assert on. Do not construct a row, + flag or field by hand and then assert the reader honours it. + +Rule B exists because the original §7 test wrote the `AgentMessage` row with +`visibility` already set and so never exercised the writer — which is how +`_post_message` shipped without stamping the field at all, leaving this section's +exemption dead code and letting private content into the public memory segment. Rule A +exists because the §5.2 symmetry test skipped the `None`-vs-set case, which is how the +`open`-policy asymmetry shipped: an uncohorted agent could act on anyone but appeared in +nobody's mate set, so it opened threads that were never answered. + +Both defects were found by a real multi-turn run, not by the suite. Both are now +mutants in `scripts/mutate_cohorts.sh`, which applies nine one-line edits to +`src/services/cohorts.py`, `src/agent/message_log.py` and `src/agent/simulation.py` and +requires each to make at least one test fail. A surviving mutant means the behaviour is +untested regardless of what the test names say. Run it after adding a cohort test; it is +offline and needs no API key. + +Sections whose normative claims had been written down but never exercised, now covered: + +| § | Claim | Where | +|---|---|---| +| 5.1 | the whole decision table, incl. empty-string agent_id and unknown visibility | `test_decision_table_row` | +| 5.3 | all four preflight inputs, 12 combinations | `test_preflight_matrix` | +| 5.4 | `open` never emits an empty gate, over six topology shapes | `test_open_policy_never_emits_an_empty_gate` | +| 6.3 | forward-only cursor: no backlog replay, and the filter is per-read not stamped at ingest | `test_filtering_is_forward_only`, `test_a_rewound_cursor_does_replay_and_the_gate_still_applies` | +| 6.3.1 | the matrix save commits exactly once, after the diff loop | `test_matrix_save_is_one_transaction` | +| 7 | the exemption through all three writers, plus every channel class stamped | `test_private_exemption_holds_for_every_write_path`, `test_every_outbound_channel_class_is_stamped` | +| 8 | grandfathered thread loses priority **and** still concludes | `test_grandfathered_thread_concludes_but_loses_priority` | +| 9 | 14 mention surroundings; indentation never reflowed | `test_strip_cases`, `test_strip_indentation_is_preserved` | +| 10.3 | the valve at 20 agents over 200 picks | `test_valve_holds_over_sustained_load` | +| 11 | membership is live, settings are cached | `test_membership_is_live_but_settings_are_cached` | +| 13.1 | the snapshot is written by `start()`, before the first turn, gate on **or** off | `test_start_computes_the_gate_and_records_a_snapshot`, `..._even_when_the_gate_is_off` | +| 14 | the CI gate itself, ordered before pytest | `test_ci_script_gates_on_alembic_before_running_tests` | +| — | 20-agent scale, mid-run activate/deactivate | `test_gate_is_correct_and_affordable_at_20_agents`, `test_{de,}activating_an_agent_mid_run_*` | + +**Measured, §10.3.** With `valve=3`, 20 agents, and two locked in a perpetual exchange, +the pair takes **150 of 200** selections and the valve forces **50** proactive picks — a +clean 3:1. With the valve effectively disabled the pair takes 200/200, which is the +check that the test has teeth. The lower bound (`pair >= 100`) is the control: a +scheduler with no reactive tier would give the pair ~2/20 of the turns and would satisfy +an upper bound on its own. + +**Measured, §14.** The `0022` round trip was run against a throwaway database: +`0022 → 0021` drops all three cohort tables, `→ 0022` re-applies clean. Then, with the +tables dropped by hand but the stamp left at `0022` — a partial upgrade — the downgrade +still succeeds and re-upgrades cleanly. That is precisely what the `if_exists=True` +guards buy, and it is the state a failed deploy leaves behind. `scripts/ci.sh` will run +the round trip when `CI_MIGRATION_DB` is set; it is off by default so the gate stays +offline. + +**Emergent behaviour, real API.** `tests/integration/test_cohort_scenarios.py` drives +real multi-turn runs and asserts on who ends up conversing with whom — the only claims +in this document that a deterministic test cannot settle. Three things make it +falsifiable, and all three come from a run that proved nothing: + +1. Every lab profile is complementary to every other, so any pair is a plausible + collaboration and the gate is the only thing that can prevent one. The earlier + version made the cohorts mutually irrelevant, so the gate-OFF baseline also produced + zero cross-cohort threads. +2. The roster is trimmed to the agents under test. Four agents over a dozen turns is not + enough for a *specific* pair to form a thread. +3. Messages the harness posts itself are recorded and excluded from every pair + measurement. Counting them would make "these two conversed" true by construction. + +And a fourth, found by executing the plan: **the scenario workspace is collapsed to a +single `#general` channel.** Phase 1 joins channels by keyword-matching the profile and +Phase 5 posts into whichever subscribed channel the model names, so across the real +seven-channel workspace two agents never landed in the same room often enough to open a +thread — measured, with su in three channels and cravatt in all seven, posting into +`#chemical-biology` and `#drug-repurposing` where su does not read. Every outcome claim +came back `INCONCLUSIVE` and *every thread had exactly one participant*. This is a +general hazard for any scenario test here: agents that cannot see each other produce the +same observation as a gate that works perfectly. Only the positive control tells the two +apart, and only per-thread diagnostics say which one you are looking at. + +`test_harness_produces_conversation_at_all` is the module's positive control: if a +permissive single-cohort run produces no conversation, every absence assertion built on +the harness is worthless, and that test is what tells you so. + +**Out of scope by instruction: Slack mirroring under an active gate.** Also not +currently possible here — no agent carries a bot token and no `SLACK_*` value is set. +Every engine test runs `slack_enabled=False` with `NullTransport`, which is the +configuration where the DB is the sole conversation store and the gate's correctness +rests entirely on read-side filtering (§9.1). + +--- + +## 16. Rollout + +1. Rename the graph "cohort" concept to "run window" (§1). Comment-only. +2. Bring the live database to `0021` on the clean `main` tree, verify the four + `agent_messages` columns (§14.5). +3. Land the CI preflight gate (§14.4) — **before** the feature, so it can catch it. +4. Land the scheduler changes alone: valve default 3, cooldown eligibility, + ratio logging, `asyncio.sleep` removal. Independently reviewable, no new + tables, immediately useful. +5. Land the cohort gate: migration `0022`, models, gated read paths (`is_bot` + keying, `_entry_allowed`), the §6.2 DB-path classifications, private-channel + exemption, grandfathering, tag hygiene, topology provenance, admin UI with audit + log and delete guard. Flag off. +6. Verify with `slack_enabled` **false** as well as true (§9.1), and on a + *resumed* run, not just a fresh one — the rebuild path (§8) is only exercised on + resume and is where the gate is blind. +7. Enable on one run with `policy="open"` and two cohorts covering the whole + roster. Watch the filter/strip/ratio counters and the grandfathered count for + one full run. +8. Only then consider `policy="isolated"`, and only with the preflight and the + isolated-agent banner in place. + +Do not enable the flag on a roster where any agent is uncohorted until step 7. + +--- + +## Appendix A — audit evidence + +All figures below were produced by execution on 2026-07-30 against +`origin/cohort-agent-isolation` @ `b00b0e6` and `main` @ `b7edcbc`, with +migrations run against throwaway clones of the live `copi` database. The live +database was not modified (verified at `0018` before and after). + +| Check | Result | +|---|---| +| Branch's own suite | 282 passed — commit message's claim reproduces exactly | +| `tests/test_cohort_isolation.py` | 20 passed | +| Merge into `main` | **clean, 0 conflicts** | +| `tests/unit` on `main` | 378 passed | +| `tests/unit` + cohort tests on merged tree | 398 passed — no regressions | +| Probes asserting v1's promises | **11 of 11 failed** | +| Probes asserting the defects | **9 of 9 passed** (both on the branch and merged) | +| `alembic heads` (merged) | `0019`, `0021` — two heads + duplicate-revision warning | +| `alembic upgrade head` at `0018` | FAILED, DB unchanged | +| `alembic upgrade heads` at `0018` | FAILED, DB unchanged | +| `alembic upgrade 0021` at `0018`, files as-shipped | **exit 0**, stamped `0021`, `cohorts` **never created** | +| same command, `add_cohorts` sorted last | **exit 0**, stamped `0021`, `agent_messages.content` **never created** | +| runtime read of `content` in that state | `UndefinedColumnError` while `alembic current` = `0021 (head)` | +| `alembic downgrade 0018` on the wedged box | crash in the wrong `0019.downgrade()`; whole chain rolled back atomically | +| `alembic check` as a guard | `Target database is not up to date` — wrong diagnosis, needs a live DB | +| Live `copi` schema | at `0018`; `content`, `slack_ts`, `slack_channel_id`, `slack_thread_ts` all MISSING | + +DB-primary interface observations (read from `main` @ `b7edcbc`): + +| Observation | Location | +|---|---| +| `MessageLog` is still in-memory append-only; DB mirrored via `_persist_cb`; all 8 engine feed sites go through `message_log.append` — the read-boundary gate remains the correct choke point | `message_log.py:1`, `:89-104` | +| `agent_messages.agent_id` is **nullable**; ingestion maps it straight to `sender_agent_id` | `0019_agent_message_content.py:49`, `agent_activity.py:82`, `simulation.py:2253` | +| `_poll_inbound_from_db` ingests all rows for the run, advancing `_pi_inbox_cursor` — filtering here would drop rows for every agent | `simulation.py:2209-2264` | +| PI DMs bypass `MessageLog` entirely, going to `PIHandler.handle_dm` | `simulation.py:2452-2498` | +| Setup order is rebuild → rebuild → rebuild, *then* loop → first `_sync_roster_from_db` | `simulation.py:396-398`, `:424`, `:446` | +| `_rebuild_agent_state` walks `message_log._entries` directly, bypassing gated accessors | `simulation.py:3302+` | +| `last_seen_cursor` advances to `time.time()` per turn; rebuild sets it to the latest message time | `simulation.py:648`, `:3479` | +| Private-channel cursor rewind exists specifically to re-scan handovers | `simulation.py:1494`, `:1554-1559` | +| `LogEntry.visibility` is carried in memory and persisted as `AgentMessage.visibility` | `message_log.py` dataclass, `agent_activity.py` | +| `NullTransport.is_connected` is `False`, so Slack pollers no-op and DB inbox is the only inbound path | `transport.py:67-99` | +| PI/admin web views read `AgentMessage` directly for display | `routers/agent_page.py`, `routers/admin.py` | + +Hypotheses tested and **refuted** — recorded so they are not re-litigated: + +- *"The merge will conflict heavily."* No. `main` rewrote 862 lines of + `simulation.py` and 111 of `message_log.py` since the fork and the merge is + still clean with all tests green. The damage is at the Alembic layer only. +- *"The gate is open for the first 30 s of a run."* No. `_last_roster_poll` + starts at `0.0` and `_sync_roster_from_db()` runs before the first + `_select_agent()`. +- *"Membership is re-queried every loop iteration."* No. The 30 s + `ROSTER_POLL_INTERVAL` early-return precedes the recompute. +- *"Some gated call sites forget to pass the gate."* No. 3 of 3 pass it. +- *"Alembic file order is filesystem-dependent, so the outcome is + non-deterministic per host."* No. `ScriptDirectory._list_py_dir` uses + `sorted(files)`; the last duplicate wins, deterministically. The hazard is that + the winner depends on filenames, and nothing in the output reveals the loser was + skipped. +- *"A failed migration could leave a half-applied schema."* No. Postgres + transactional DDL plus a single `begin_transaction()` around the whole chain + meant every failure path rolled back completely, data intact. + +## Appendix B — retired v1 decisions + +Kept so the reasoning is not rediscovered: + +- **Min-heap selection** (v1 §6) — retired. The problem was thread latency, not + starvation; priority tiers address it directly. Revisit only if measurement + shows proactive starvation under §10.3's 3:1 valve. +- **Global semaphore / `concurrent_turns`** (v1 §6) — retired. Brings duplicate + pair initiation, single-token posting contention, and non-deterministic state + mutation. v1 §7 existed solely to patch the first of those. +- **Concurrent pair-initiation guard** (v1 §7) — retired with concurrency. +- **`Agent.cohort_ids` + `can_interact()`** (v1 §Agent Changes) — retired in + favour of a precomputed `allowed_sender_ids` set, which is cheaper and has one + evaluation point. The *semantics* v1 attached to `can_interact` are restored in + §5. +- **Separate 60 s `COHORT_RESYNC_INTERVAL`** — retired; the 30 s roster tick + already covers it. +- **O(n²) cost framing** — retired as factually wrong under the sequential + scheduler; replaced by §3. diff --git a/specs/cohort-system.md b/specs/cohort-system.md index 0d65b99..8ef9c21 100644 --- a/specs/cohort-system.md +++ b/specs/cohort-system.md @@ -1,438 +1,22 @@ # Cohort System Specification -## Overview - -A cohort is a named group of agents whose members are permitted to interact with each other during simulation. The purpose is purely practical: prevent agents from spending LLM turns scanning, activating threads with, or tagging agents they will never productively engage. Cohorts are orthogonal to Slack channels — channel subscriptions remain unchanged; cohort membership only gates whether one agent will *act on* another agent's activity. - -Agents may belong to any number of cohorts. Cohort assignments are admin-managed and can change while a simulation is running. Interaction limits (thread count, proposal caps, budgets) remain per-agent and are shared across all cohorts an agent belongs to. - ---- - -## Goals - -- Skip Phase 2 scan evaluation of posts from non-cohort agents (save Sonnet calls) -- Skip Phase 3 thread activation from non-cohort agents (save CPU + state bloat) -- Skip Phase 5 tagging or replying to non-cohort agents (save Opus calls) -- Run N turns concurrently via a global semaphore for predictable API cost at any agent list size -- Ensure fair turn distribution across all agents via min-heap selection -- Allow membership to change mid-run without requiring a restart - ---- - -## Data Model - -### New Table: `cohorts` - -```sql -CREATE TABLE cohorts ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - name TEXT NOT NULL UNIQUE, - description TEXT, - created_by UUID REFERENCES users(id) ON DELETE SET NULL, - created_at TIMESTAMP WITH TIME ZONE DEFAULT now() -); -``` - -- `name`: short slug-style identifier (e.g. `"pilot-wave-1"`, `"structural-cohort"`). Unique, immutable after creation. -- `description`: optional free-text note for admin reference. -- `created_by`: FK to the admin user who created it; nullable (SET NULL on user delete). - -### New Table: `cohort_memberships` - -```sql -CREATE TABLE cohort_memberships ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - cohort_id UUID NOT NULL REFERENCES cohorts(id) ON DELETE CASCADE, - agent_id TEXT NOT NULL, - added_by UUID REFERENCES users(id) ON DELETE SET NULL, - added_at TIMESTAMP WITH TIME ZONE DEFAULT now(), - UNIQUE (cohort_id, agent_id) -); -``` - -- `agent_id`: matches `AgentRegistry.agent_id` (string, e.g. `"su"`, `"wiseman"`). No FK enforced — agent records may not exist at table creation time; the application validates at join time. -- Composite unique constraint prevents duplicate membership. -- Cascade delete: removing a cohort removes all its memberships. - -### Migration - -File: `alembic/versions/0023_add_cohorts.py` - -```python -def upgrade(): - op.create_table("cohorts", ...) - op.create_table("cohort_memberships", ...) - op.create_index("ix_cohort_memberships_cohort_id", "cohort_memberships", ["cohort_id"]) - op.create_index("ix_cohort_memberships_agent_id", "cohort_memberships", ["agent_id"]) - -def downgrade(): - op.drop_table("cohort_memberships") - op.drop_table("cohorts") -``` - -### SQLAlchemy Models - -`src/models/cohort.py`: - -```python -class Cohort(Base): - __tablename__ = "cohorts" - id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) - name: Mapped[str] = mapped_column(unique=True) - description: Mapped[str | None] - created_by: Mapped[UUID | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL")) - created_at: Mapped[datetime] = mapped_column(default=func.now()) - memberships: Mapped[list["CohortMembership"]] = relationship(back_populates="cohort", cascade="all, delete-orphan") - -class CohortMembership(Base): - __tablename__ = "cohort_memberships" - id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4) - cohort_id: Mapped[UUID] = mapped_column(ForeignKey("cohorts.id", ondelete="CASCADE")) - agent_id: Mapped[str] - added_by: Mapped[UUID | None] = mapped_column(ForeignKey("users.id", ondelete="SET NULL")) - added_at: Mapped[datetime] = mapped_column(default=func.now()) - cohort: Mapped["Cohort"] = relationship(back_populates="memberships") -``` - -Export from `src/models/__init__.py` alongside existing models. - ---- - -## Agent Changes - -### `src/agent/agent.py` - -Add one field to `Agent.__init__`: - -```python -self.cohort_ids: set[str] = set() # populated by SimulationEngine at startup and on resync -``` - -Add one helper method: - -```python -def can_interact(self, other: "Agent") -> bool: - """True if the two agents share at least one cohort (or if either has no cohort assignments).""" - if not self.cohort_ids or not other.cohort_ids: - return True # uncohorted agents interact with everyone — backward-compatible default - return bool(self.cohort_ids & other.cohort_ids) -``` - -The fallback `return True` when either agent has no cohorts assigned preserves all-vs-all behaviour for agents not yet assigned to any cohort, preventing accidental silencing. - ---- - -## Simulation Engine Changes - -### `src/agent/main.py` - -#### 1. Cohort Loading at Startup - -After agents are loaded and before the main loop, query cohort memberships: - -```python -async def _load_cohort_memberships(self): - async with self._session_factory() as db: - rows = await db.execute( - select(CohortMembership.agent_id, CohortMembership.cohort_id) - ) - # Clear and rebuild - for agent in self.agents.values(): - agent.cohort_ids = set() - for agent_id, cohort_id in rows: - if agent_id in self.agents: - self.agents[agent_id].cohort_ids.add(cohort_id) - - self._last_cohort_sync = time.time() - logger.info("Cohort memberships loaded for %d agents", sum(1 for a in self.agents.values() if a.cohort_ids)) -``` - -No index structure is needed — the interaction gate operates purely via `agent.cohort_ids` set intersection at the point of interaction. Turn dispatch is global and cohort-unaware (see Section 6). - -#### 2. Dynamic Membership Resync - -Every 60 seconds (checked at the top of each main-loop round), re-run `_load_cohort_memberships()` and rebuild `_cohort_members`. This is a full replace, not a diff — simple and correct. - -```python -COHORT_RESYNC_INTERVAL = 60 # seconds - -if time.time() - self._last_cohort_sync >= COHORT_RESYNC_INTERVAL: - await _load_cohort_memberships() - _rebuild_cohort_index() -``` - -Resync only updates `agent.cohort_ids` and `_cohort_members`. It does not touch `AgentState` or close any active threads — existing open threads between agents who have since been removed from a shared cohort are allowed to conclude naturally. - -#### 3. Interaction Gate — Phase 2 - -In `_phase2_scan_filter()`, filter incoming posts before building the LLM prompt: - -```python -new_posts = [ - p for p in new_posts - if self._sender_can_interact(agent, p.sender_agent_id) -] -``` - -Where: - -```python -def _sender_can_interact(self, agent: Agent, sender_id: str | None) -> bool: - if sender_id is None: - return True # PI/human message — always show - sender = self.agents.get(sender_id) - if sender is None: - return True # unknown sender — don't filter - return agent.can_interact(sender) -``` - -#### 4. Interaction Gate — Phase 3 - -In `_phase3_activate_threads()`, tag-based and reply-based activation both check: - -```python -sender = self.agents.get(entry.sender_agent_id) -if sender and not agent.can_interact(sender): - continue # skip activation — not a cohort-mate -``` - -This applies before any other checks (thread cap, thread participation rules, etc.) to fail fast. - -#### 5. Interaction Gate — Phase 5 - -In `_phase5_new_post()`, when filtering `available_posts`: - -```python -sender = self.agents.get(post.sender_agent_id) -if sender and not agent.can_interact(sender): - agent.state.interesting_posts = [ - p for p in agent.state.interesting_posts if p.post_id != post.post_id - ] - continue # prune stale post — sender is no longer a cohort-mate -``` - -When the LLM response names a `tagged_agent` for a new top-level post: - -```python -if tagged_agent: - target = self.agents.get(tagged_agent) - if target and not agent.can_interact(target): - logger.debug("%s: cohort gate blocked tag of %s in phase5", agent.agent_id, tagged_agent) - return -``` - -#### 6. Turn Selection: Min-Heap + Global Semaphore - -Replace the current O(n) weighted-random `_select_agent()` with a **min-heap keyed by `last_selected`** and a **global semaphore of width `concurrent_turns`**. - -**Why min-heap over weighted random:** -The current weighted-random gives probabilistic fairness but can starve agents at large list sizes, particularly when `phase5_skip_probability` is non-zero (fast no-op turns let an agent re-enter the lottery immediately). A min-heap guarantees the longest-waiting eligible agent always gets the next slot — O(log n) selection, deterministic fairness. - -**Selection and dispatch:** - -```python -import heapq - -def _build_heap(self) -> list[tuple[float, Agent]]: - now = time.time() - return [ - (a.state.last_selected, a) - for a in self.agents.values() - if not a.is_paused - and self._agent_within_budget(a) - and (now - a.state.last_selected) >= settings.turn_delay_seconds - ] - -async def _run_concurrent_turns(self) -> bool: - heap = self._build_heap() - if not heap: - return False - - heapq.heapify(heap) - n = min(settings.concurrent_turns, len(heap)) - selected = [heapq.heappop(heap)[1] for _ in range(n)] - - results = await asyncio.gather( - *[self._run_turn(agent) for agent in selected], - return_exceptions=True, - ) - - did_any_work = False - for agent, result in zip(selected, results): - agent.state.last_selected = time.time() - if isinstance(result, Exception): - logger.exception("Turn error for %s", agent.agent_id) - elif result: - did_any_work = True - - return did_any_work -``` - -The main loop calls `_run_concurrent_turns()` each iteration and uses `did_any_work` to drive the existing idle-backoff logic unchanged. - -**Slack polling** continues once per round, before `_run_concurrent_turns()`, as a single sequential operation. - -**`_last_llm_caller` guard:** This guard exists to prevent the same agent from making back-to-back LLM calls in the sequential model. It is superseded by the min-heap + per-agent cooldown (`turn_delay_seconds` eligibility check) and should be removed from the concurrent path. The min-heap naturally pushes a just-selected agent to the bottom of the queue; the cooldown makes them ineligible until the delay has elapsed. - -#### 7. Phase 5 Concurrent Initiation Guard - -With N turns running concurrently, two agents can independently decide to start a new thread with each other in the same round (both see `has_pending_reply=False` and neither has an active thread with the other yet). Track in-flight pair initiations to prevent duplicate thread creation: - -```python -self._initiating_pairs: set[frozenset[str]] = set() -``` - -In `_phase5_new_post()`, before posting a reply that opens a new thread toward `target_agent_id`: - -```python -pair = frozenset([agent.agent_id, target_agent_id]) -if pair in self._initiating_pairs: - logger.debug("%s: concurrent initiation guard blocked duplicate thread with %s", agent.agent_id, target_agent_id) - return - -self._initiating_pairs.add(pair) -try: - await self._post_message(...) - # activate thread ... -finally: - self._initiating_pairs.discard(pair) -``` - -The pair is removed once the thread is activated (or on failure). Note: Phase 4 back-and-forth replies are safe without this guard — `has_pending_reply` is a logical baton held by only one side at a time, so two agents cannot both have a pending reply to each other simultaneously. - ---- - -## Admin Interface - -### Routes - -All routes are added to `src/routers/admin.py` under the `/admin/cohorts` prefix, protected by the existing `get_admin_user` dependency. - -| Method | Path | Description | -|--------|------|-------------| -| GET | `/admin/cohorts` | List all cohorts with member counts | -| POST | `/admin/cohorts/create` | Create a new cohort | -| GET | `/admin/cohorts/{cohort_id}` | Cohort detail: members, audit log | -| POST | `/admin/cohorts/{cohort_id}/delete` | Delete cohort (cascades memberships) | -| POST | `/admin/cohorts/{cohort_id}/add-agent` | Add an agent to the cohort | -| POST | `/admin/cohorts/{cohort_id}/remove-agent` | Remove an agent from the cohort | - -POST routes redirect back to the referring page on success and render an inline error on failure (same pattern as existing admin routes). - -### Cohort List Page — `GET /admin/cohorts` - -Template: `templates/admin/cohorts.html` - -**Header:** "Cohorts" with a "New Cohort" button (opens inline form or modal). - -**Create form** (inline, collapsed by default): -- `name` (text input, required) — validated: lowercase, alphanumeric + hyphens only, max 48 chars -- `description` (textarea, optional) -- Submit → `POST /admin/cohorts/create` - -**Table: All Cohorts** - -| Column | Notes | -|--------|-------| -| Name | Link to detail page | -| Description | Truncated at 80 chars | -| Members | Count of current memberships | -| Created by | Admin user name | -| Created at | Date | -| Actions | Delete button (with confirmation; disabled if cohort has active members) | - -If no cohorts exist: empty state with "No cohorts yet. Create one above." - -### Cohort Detail Page — `GET /admin/cohorts/{cohort_id}` - -Template: `templates/admin/cohort_detail.html` - -**Header:** Cohort name + description. Delete button (top right, requires confirmation prompt via `data-confirm` attribute; only shown if member count is 0, otherwise disabled with tooltip "Remove all members first"). - -**Section: Members** - -Table of current members: - -| Column | Notes | -|--------|-------| -| Agent ID | e.g. `su`, `wiseman` | -| Bot Name | e.g. `SuBot` | -| PI Name | e.g. `Andrew Su` | -| Agent Status | `active` / `suspended` / `pending` (from AgentRegistry) | -| Added by | Admin user name | -| Added at | Date | -| Actions | "Remove" button → `POST /admin/cohorts/{cohort_id}/remove-agent` with `agent_id` | - -**Section: Add Agent** - -Dropdown of all agents *not already in this cohort*, populated from AgentRegistry. Only active agents are shown by default; a checkbox toggle shows suspended/pending agents as well. - -``` -[ Select agent ▼ ] [ Add to Cohort ] -``` - -`POST /admin/cohorts/{cohort_id}/add-agent` body: `{ agent_id: "su" }` - -If the selected agent already belongs to this cohort, return a 400 with inline error "Agent is already a member." - -**Section: Agent Cohort Map (read-only)** - -Summary table showing all active agents and which cohorts they currently belong to, for cross-reference: - -| Agent | Cohorts | -|-------|---------| -| SuBot | pilot-wave-1, structural | -| WisemanBot | pilot-wave-1 | -| LotzBot | *(none)* | - -This section is static (no editing — use individual cohort pages to manage membership). - -### Navigation - -Add "Cohorts" to the existing admin sidebar nav alongside Agents, Users, Activity, etc. - ---- - -## Configuration - -### New settings (`src/config.py`) - -```python -concurrent_turns: int = 3 # max simultaneous agent turns; overridden by active_thread_threshold at runtime -``` - -At engine startup, `concurrent_turns` is clamped to `max(concurrent_turns, active_thread_threshold)`. This keeps the two levers in proportion: if an admin raises the thread threshold to allow more simultaneous conversations, the concurrent turn capacity rises with it automatically. The `concurrent_turns` setting therefore acts as a floor, not a ceiling. - -The cohort resync interval is hardcoded as `COHORT_RESYNC_INTERVAL = 60` seconds in the engine. It can be promoted to `Settings` if operational tuning is needed. - -### `turn_delay_seconds` — Behavior Change - -**Current behavior (to be removed):** `simulation.py:360-361` applies `asyncio.sleep(turn_delay_seconds)` at the end of every productive main-loop iteration. This is a **global pause** — no Slack polling, no other agents, nothing runs during the sleep. It is 0.0 by default and has no per-agent targeting. - -**New behavior:** `turn_delay_seconds` becomes a **per-agent cooldown** enforced at selection time inside `_build_heap()`: - -```python -and (now - a.state.last_selected) >= settings.turn_delay_seconds -``` - -An agent that just completed a turn is ineligible until the cooldown has elapsed. All other agents are unaffected. The `asyncio.sleep(settings.turn_delay_seconds)` call in `simulation.py` is removed. - -This preserves the original intent (throttle individual agent tempo) while composing correctly with concurrent dispatch — N slots can stay busy while a recently-active agent sits out its cooldown. - ---- - -## Backward Compatibility - -- Agents with no cohort memberships are grouped into `"__uncohorted__"` and continue to interact with all other uncohorted agents. This means a simulation with zero cohorts defined behaves identically to the current all-vs-all system. -- `Agent.can_interact()` returns `True` when either agent has an empty `cohort_ids` set, so partially-cohorted simulations (some agents assigned, some not) do not silently break. -- No existing tables, models, or routes are modified. - ---- - -## Out of Scope - -- Agent-visible cohort concept: agents do not know which cohort a conversation was initiated from; threads are indistinguishable. -- PI-managed cohorts: only admins create and delete cohorts. PIs cannot request cohort changes. -- Per-cohort budgets or limits: all limits remain per-agent and are shared across cohorts. -- Cohort-scoped message history or separate Slack workspaces per cohort. -- Time-bounded cohort memberships (automatic expiry). +**Superseded.** This file described cohort system v1, which was never the design +that shipped. It disagreed with the implementation on the migration filename +(`0023_add_cohorts.py` vs the shipped `0022`), the table count (2 vs 3 — it +omitted `cohort_audit_events`), and turn selection (min-heap plus a global +semaphore vs the reactive/proactive weighted selector in +`Simulation._select_agent`). It described none of the mechanisms that shipped: +`cohort_isolation_enabled`, `cohort_default_policy`, thread grandfathering, +gate preflight, topology snapshots, audit events, or reactive scheduling. + +**The current specification is [`specs/cohort-system-v2.md`](cohort-system-v2.md).** + +31 comments across 16 files in `src/`, `tests/`, `scripts/`, `alembic/` and +`templates/` cite it as `.notes/cohort-system-v2.md §N`, the path it was written +at before it was promoted. **Read those as `specs/cohort-system-v2.md §N`** — the +section numbering is identical, and all 15 distinct cited sections (§2, §4.2, §5, +§5.1, §5.2, §6, §6.2, §7, §8, §9, §10.3, §12, §13.1, §14, §15) resolve to real +headings in the tracked copy. + +`.notes/` stays ignored, so the tracked copy under `specs/` is the only one that +ships. If you edit the spec, edit this one. diff --git a/specs/local-db-conversations.md b/specs/local-db-conversations.md new file mode 100644 index 0000000..9248b3f --- /dev/null +++ b/specs/local-db-conversations.md @@ -0,0 +1,174 @@ +# Local-DB-Backed Conversations (Slack as an optional mirror) + +Status: in progress. Companion to `specs/agent-system.md` and +`specs/privacy-and-channel-visibility.md`. + +## Motivation + +Historically the agent simulation treated **Slack as the primary durable store +of conversation content**. The in-memory `MessageLog` (`src/agent/message_log.py`) +held message text only for the life of the process and was **rebuilt from Slack +history on every startup** (`SimulationEngine._rebuild_state_from_slack`). The +database was a metadata-only sibling: `agent_messages` stored `message_length` +but no content, and only the agent's *own* posts were recorded. With Slack off, +the system could not reconstruct any conversation and a restart lost all history. + +This spec makes the local PostgreSQL database the **single source of truth** for +conversations. The simulation must run identically with **all Slack API access +disabled**. When Slack is enabled it is a redundant, bidirectional mirror/view: + +- **Outbound:** DB-origin messages are posted to Slack for human viewing. +- **Inbound:** human/PI Slack messages are written into the DB. + +The DB reproduces the Slack-provided semantics the engine relies on — channels, +threads, membership/permissions (`public` vs `collab_private`), and PI↔bot DMs — +by **reusing the existing schema** wherever possible. + +## Core design rules + +1. **Canonical id = the id a message is born with.** A DB-origin message + (Slack-off, or an agent post in mirror mode) gets a locally-minted, ts-shaped + id from `mint_ts()`. A Slack-origin message keeps its Slack `ts` as canonical. + In pure Slack-on mode `message_ts == slack_ts`, so behavior is identical to the + pre-change system. Every structure keyed by ts (`PostRef.post_id`, + `ThreadState.thread_id`, `_poll_cursors`, `ThreadDecision.thread_id`, + `MessageLog._by_ts`) is unchanged. + + **Corollary: never hand a canonical id to Slack.** Slack threads on the + *root's* `ts`, which equals the canonical `thread_ts` only when the root was + born on Slack. `_slack_parent_ts()` translates canonical → Slack via the root + entry's `slack_ts`, and `_post_message` skips the mirror entirely (DB-only, + with a warning) when the root has no Slack presence, rather than posting + against an id Slack has never seen. This is what makes enabling Slack + mid-conversation degrade safely instead of detaching or erroring. The mapping + is restored on rebuild, so it survives a restart. The web process has no + `MessageLog`, so `private_channels._slack_parent_ts_from_db()` performs the + same translation from `agent_messages` for the private-channel migration's + origin-thread close marker. + + **`slack_ts` is the only evidence of Slack presence; a NULL means "not on + Slack".** It is never inferred from the channel id. Inferring ("a row in a + real Slack channel was born on Slack, so its canonical id is its Slack ts") + looks reasonable for pre-Stage-6 history but is unsound: a DB-origin message + can carry a real Slack `channel_id` too — a PI message from the web inbox + resolves it from the `agent_channels` row, and so does an agent post whose + mirror failed. Both mint a *local* id, and inferring promotes it to a Slack ts + Slack never issued, which then goes out as a `chat.postMessage` `thread_ts` + and orphans the reply. Legacy rows are repaired once by + `scripts/backfill_slack_ts.py`, which asks Slack which timestamps exist rather + than assuming; run it before deploying on a workspace with pre-Stage-6 history. + +2. **`mint_ts()` is monotonic and unique — across processes, not just within + one.** Ids are carried as integer microseconds (never round-tripped through a + float, which cannot hold microsecond precision at current epoch magnitudes) + and strictly advance; the high-water mark is seeded at rebuild from + `max(posted_at)` so minted ids sort after restored history. This preserves + `posted_at = float(ts)` ordering. Three processes mint into the same run — the + engine, the web app and GrantBot — so each minter also owns a **writer slot**: + ids are quantized to `WRITER_SLOT_MODULUS` (100 µs) and the writer id occupies + the low microsecond digits, giving every writer its own residue class. Without + this, two processes minting in the same microsecond produce the identical id, + and the `uq_agent_messages_run_ts` conflict handler resolves it by *dropping* + one message — unrecoverable now that the DB is the only durable store. Writer + ids are claimed at process entry with `set_default_writer_id()` + (`src/agent/ids.py`). The DB constraint remains the backstop. + +3. **Persist at the single chokepoint `MessageLog.append`.** Peer, human/PI, and + reopen messages reach state only via `append`. A persist callback there + (mirroring `set_bot_name_map`) captures all senders and keeps `message_log.py` + DB-agnostic. DMs never enter `MessageLog`, so they persist separately. + +4. **`append` is idempotent** (skip in-memory add and callback when `ts` is + already present) — safe only because `mint_ts()` guarantees uniqueness. + +5. **Reuse, don't replace.** Extend `agent_messages` rather than add a parallel + table. The `visibility` model, `private_channel_members`, `_visibility_permits`, + thread participant rules, and `_sync_private_channels_from_db` all port directly. + +## Schema + +### `agent_messages` (extended — migration 0019) + +Adds, all NOT NULL with server defaults so existing rows survive: +`content Text ''`, `sender_name String(100) ''`, `is_bot Boolean true`, +`posted_at Float 0` (ordering key), and nullable mirror columns +`slack_ts String(50)`, `slack_channel_id String(100)`, `slack_thread_ts String(50)`. +`agent_id` is relaxed to nullable and read as `sender_agent_id` (NULL = human/PI). +`message_ts` is the canonical id. + +Indexes: `UNIQUE(simulation_run_id, message_ts)`, +`INDEX(simulation_run_id, posted_at)`, +`INDEX(simulation_run_id, channel_name, posted_at)`, +partial `INDEX(simulation_run_id, slack_ts) WHERE slack_ts IS NOT NULL`. + +### `pi_dm_messages` (new — migration 0020) + +`id`, `simulation_run_id` (FK cascade), `agent_id`, `pi_user_id` (Slack user id +Slack-on; `local:` off), `direction` enum(`inbound`,`outbound`), +`content`, `sender_name`, `ts` (canonical), `slack_ts` (nullable), `posted_at`, +`created_at`. Indexes `(simulation_run_id, agent_id, posted_at)` and +`(simulation_run_id, direction, posted_at)`. + +### Channels + +No schema change. Slack-off stores `local:` in `agent_channels.channel_id`. +Public subscriptions stay re-derived from profile keywords at seeding; a +`channel_subscriptions` table is deferred until hand-edited/agent-created public +subscriptions must survive a restart independent of re-derivation. + +## Transport abstraction + +`src/agent/transport.py` defines a `Transport` `Protocol` covering the exact +method set the engine calls on `AgentSlackClient`: + +- outbound: `post_message`, `send_dm`, `create_channel`, `create_private_channel`, + `invite_to_channel`, `join_channel`, `list_channels`, `open_dm_channel` +- inbound: `poll_channel_messages`, `get_thread_replies`, `get_all_thread_replies`, + `get_full_channel_history`, `poll_dm_messages` +- identity: `connect`, `is_connected`, `bot_user_id`, `resolve_user_name`, + `is_bot_user` + +`SlackTransport` is today's `AgentSlackClient` conformed. `NullTransport` reports +`is_connected=False` / `bot_user_id=None` (so existing +`if client and client.is_connected` branches take the no-op path), returns a +minted-id dict from outbound calls, and `[]` from inbound calls. + +`slack_enabled` (config + CLI) auto-detects from the presence of ≥1 valid token +with an explicit override; `--mock`/no-token ⇒ Slack-off. + +## PI interaction without Slack + +When Slack is off, PIs interact through a web interface that **writes inbound +rows** (`agent_messages` with `is_bot=false`/`sender_agent_id=null`, or +`pi_dm_messages` `direction='inbound'`). The engine's `_poll_pi_inbox_from_db()` +reads those new rows each tick and routes them through the existing PI-handling +logic (`_check_pi_proposal_review`, `has_pi_directive`, thread reopen, `@bot` tag +→ `handle_channel_tag`/`handle_dm`). This is the convergence point: the Slack +mirror's inbound side and the Slack-off PI path are the same DB reader. Identity +is `AgentRegistry.user_id` rather than `slack_user_id`. + +## Rollout (staged, each independently shippable) + +1. Content persistence + DB rebuild (`_rebuild_state_from_db`; demote + `_rebuild_state_from_slack` to a Slack-gated reconcile). +2. Local id minting (`mint_ts`, remove `mock_ts` constant, `local:` channel ids). +3. Transport abstraction + `slack_enabled` + `_poll_pi_inbox_from_db`. +4. Slack-less private-channel migration branch. Both branches persist the handover + (posts + origin-thread close marker) through one helper, + `private_channels._add_handover_message`, so the refinement channel is + reconstructable from the DB alone whether or not Slack was involved. +5. PI web interface. +6. Secondary Slack posters guarded by `slack_enabled`; outbound mirror write-back + (records `slack_ts`; reconcile dedups on `slack_ts`). +7. One-time Slack→DB backfill (`scripts/backfill_slack_history_to_db.py`). + +## Verification + +- Slack-off boot: `python -m src.agent.main --mock --fresh --max-runtime 1` + starts, runs `_rebuild_state_from_db`, seeds `local:` channels, persists content. +- Continuity: two Slack-off runs — resume rebuilds the log with content and + active threads intact. +- Parity: Slack-on produces identical DB/log state and `message_ts == slack_ts`. +- Existing suites stay green (`tests/test_message_log.py`, + `test_simulation_logic.py`, `test_private_channel_migration.py`, + `test_roster_sync.py`, `test_privacy_scoping.py`, `test_thread_not_found.py`). diff --git a/src/agent/agent.py b/src/agent/agent.py index 5eee687..13be714 100644 --- a/src/agent/agent.py +++ b/src/agent/agent.py @@ -2,17 +2,19 @@ import logging import re +import time from pathlib import Path -from typing import Any +from src.agent.post_types import render_menu from src.agent.prompt_safety import delimit -from src.agent.state import AgentState, PostRef, ThreadState +from src.agent.roles import DEFAULT_ROLE, load_role, resolve_prompt_path +from src.agent.state import AgentState, ThreadState +from src.agent.thread_guidance import phase4_guidance from src.models.agent_activity import VISIBILITY_COLLAB_PRIVATE, VISIBILITY_PUBLIC logger = logging.getLogger(__name__) PROFILES_DIR = Path("profiles") -PROMPTS_DIR = Path("prompts") # Matches a bare DOI. The character class deliberately excludes the delimiters # that wrap DOIs in Slack posts (whitespace, quotes, angle brackets from @@ -64,10 +66,12 @@ class Agent: Holds identity, profiles, and per-simulation mutable state. """ - def __init__(self, agent_id: str, bot_name: str, pi_name: str): + def __init__(self, agent_id: str, bot_name: str, pi_name: str, + role: str = DEFAULT_ROLE): self.agent_id = agent_id # e.g., "su" self.bot_name = bot_name # e.g., "SuBot" self.pi_name = pi_name # e.g., "Andrew Su" + self.role = role # e.g., "pi_lab" — selects prompt/role overrides self._public_profile: str | None = None self._private_profile: str | None = None self._public_working_memory: str | None = None # cached public memory segment @@ -76,6 +80,22 @@ def __init__(self, agent_id: str, bot_name: str, pi_name: str): self.api_call_count: int = 0 self.message_count: int = 0 self.state = AgentState() + # Cohort interaction gate: set of agent_ids this agent may act on (its + # cohort-mates), or None when isolation is disabled (all-vs-all). + # Recomputed each roster sync by SimulationEngine. See specs/cohort-system.md. + self.allowed_sender_ids: set[str] | None = None + + def record_api_call(self, now: float | None = None) -> None: + """Record one LLM call against both the lifetime counter and the + sliding-window ledger. + + The single write point for both. Every call site must use this rather + than bumping ``api_call_count`` directly — a site that bumps only the + counter is invisible to the rate limiter, and a site that appends only to + the ledger corrupts ``SimulationRun.total_api_calls``. + """ + self.api_call_count += 1 + self.state.call_times.append(time.time() if now is None else now) # ------------------------------------------------------------------ # Profile properties (cached, loaded from disk) @@ -184,34 +204,12 @@ def build_system_prompt( ``channel_id`` is also injected and a Private Channel Rules block is appended. See specs/privacy-and-channel-visibility.md §G1, §G4. """ - base_prompt = self._load_file( - PROMPTS_DIR / "agent-system.md", - _default_system_prompt(), + return self._compose_system_prompt( + include_memory=True, + include_lab_directory=True, + visibility=visibility, + channel_id=channel_id, ) - lab_directory_section = "" - if self._lab_directory: - lab_directory_section = f""" -## Other Labs' Recent Publications -Use these to reference other labs' work in conversations. Include links when citing. -{self._lab_directory} -""" - working_memory_text = self._compose_working_memory(visibility, channel_id) - private_rules = PRIVATE_CHANNEL_RULES if visibility == VISIBILITY_COLLAB_PRIVATE else "" - return f"""{base_prompt} - -## Your Identity -You are **{self.bot_name}**, the AI agent representing the {self.pi_name} lab at Scripps Research. -Your agent ID is "{self.agent_id}". When communicating, represent your lab professionally. - -## Your Lab Profile (Public) -{self.public_profile} - -## Your Private Instructions -{self.private_profile} - -## Your Working Memory -{working_memory_text} -{lab_directory_section}{private_rules}""" def build_scan_system_prompt(self) -> str: """Build a lightweight system prompt for scan/filter phases. @@ -219,21 +217,10 @@ def build_scan_system_prompt(self) -> str: Omits working memory and lab directory — scan only needs identity, research focus, and private priorities to judge relevance. """ - base_prompt = self._load_file( - PROMPTS_DIR / "agent-system.md", - _default_system_prompt(), + return self._compose_system_prompt( + include_memory=False, + include_lab_directory=False, ) - return f"""{base_prompt} - -## Your Identity -You are **{self.bot_name}**, the AI agent representing the {self.pi_name} lab at Scripps Research. -Your agent ID is "{self.agent_id}". When communicating, represent your lab professionally. - -## Your Lab Profile (Public) -{self.public_profile} - -## Your Private Instructions -{self.private_profile}""" def build_thread_reply_system_prompt( self, @@ -250,26 +237,81 @@ def build_thread_reply_system_prompt( which memory segment is injected and whether the Private Channel Rules block is appended. """ - base_prompt = self._load_file( - PROMPTS_DIR / "agent-system.md", - _default_system_prompt(), + return self._compose_system_prompt( + include_memory=True, + include_lab_directory=False, + visibility=visibility, + channel_id=channel_id, ) - working_memory_text = self._compose_working_memory(visibility, channel_id) + + def _load_prompt(self, filename: str, default: str) -> str: + """Load a prompt file honouring this agent's role override. + + See src/agent/roles.py: ``pi_lab`` (the default role) always falls + through to the global ``prompts/{filename}`` — that fallthrough *is* + what keeps existing agents byte-identical after this method's + introduction. + """ + return self._load_file(resolve_prompt_path(self.role, filename), default) + + def _render_identity(self) -> str: + """Render the '## Your Identity' block for this agent.""" + template = self._load_prompt("identity.md", _DEFAULT_IDENTITY) + # str.replace, NOT str.format: profiles/role files may contain bare + # curly braces (e.g. "budget is {tight}") that must not be treated as + # format fields. + return ( + template.replace("{bot_name}", self.bot_name) + .replace("{pi_name}", self.pi_name) + .replace("{agent_id}", self.agent_id) + ) + + def _compose_system_prompt( + self, + *, + include_memory: bool, + include_lab_directory: bool, + visibility: str = VISIBILITY_PUBLIC, + channel_id: str | None = None, + ) -> str: + """Assemble a system prompt from the shared sections. + + This is the single composer behind build_system_prompt, + build_scan_system_prompt, and build_thread_reply_system_prompt — the + include_memory/include_lab_directory flags reproduce each builder's + original section set byte-for-byte (see the callers below). + """ + base_prompt = self._load_prompt("agent-system.md", _default_system_prompt()) + identity = self._render_identity() private_rules = PRIVATE_CHANNEL_RULES if visibility == VISIBILITY_COLLAB_PRIVATE else "" - return f"""{base_prompt} -## Your Identity -You are **{self.bot_name}**, the AI agent representing the {self.pi_name} lab at Scripps Research. -Your agent ID is "{self.agent_id}". When communicating, represent your lab professionally. + header = f"""{base_prompt} + +{identity} ## Your Lab Profile (Public) {self.public_profile} ## Your Private Instructions -{self.private_profile} +{self.private_profile}""" + + if not include_memory: + return header + + working_memory_text = self._compose_working_memory(visibility, channel_id) + memory_block = f"\n\n## Your Working Memory\n{working_memory_text}" + + if include_lab_directory: + lab_directory_section = "" + if self._lab_directory: + lab_directory_section = f""" +## Other Labs' Recent Publications +Use these to reference other labs' work in conversations. Include links when citing. +{self._lab_directory} +""" + return f"{header}{memory_block}\n{lab_directory_section}{private_rules}" -## Your Working Memory -{working_memory_text}{private_rules}""" + return f"{header}{memory_block}{private_rules}" def _compose_working_memory( self, @@ -308,8 +350,8 @@ def build_phase2_scan_prompt(self, new_posts: list[dict[str, str]]) -> tuple[str Returns (system_prompt, messages). """ system_prompt = self.build_scan_system_prompt() - phase2_template = self._load_file( - PROMPTS_DIR / "phase2-scan-filter.md", + phase2_template = self._load_prompt( + "phase2-scan-filter.md", "Evaluate posts and return JSON with selected_post_ids.", ) @@ -338,8 +380,8 @@ def build_phase2_scan_prompt(self, new_posts: list[dict[str, str]]) -> tuple[str def build_phase2_prune_prompt(self) -> tuple[str, list[dict]]: """Build system + messages for Phase 2 prune.""" system_prompt = self.build_scan_system_prompt() - prune_template = self._load_file( - PROMPTS_DIR / "phase2-prune.md", + prune_template = self._load_prompt( + "phase2-prune.md", "Prune interesting_posts to ≤20. Return JSON with keep_post_ids.", ) @@ -380,60 +422,23 @@ def build_phase4_prompt( system_prompt = self.build_thread_reply_system_prompt( visibility=visibility, channel_id=channel_id, ) - phase4_template = self._load_file( - PROMPTS_DIR / "phase4-thread-reply.md", + phase4_template = self._load_prompt( + "phase4-thread-reply.md", "Compose a thread reply.", ) - # Thread phase guidance - if thread.message_count <= 4: - thread_phase = "EXPLORE" - phase_guidance = ( - "You are in the EXPLORE phase. Share relevant specifics from your lab's recent work. " - "Ask clarifying questions about the other lab's capabilities. Use retrieve_profile and " - "retrieve_abstract tools to learn more. Do NOT propose a full collaboration yet." - ) - elif thread.message_count <= 11: - thread_phase = "DECIDE" - phase_guidance = ( - "You are in the DECIDE phase. Narrow the scope: is there genuine complementarity? " - "Can you name a specific first experiment? If yes, build toward a :memo: Summary proposal. " - "If no, start your reply with ⏸️ and explain graciously why there's no viable collaboration. " - "It is OK to conclude with no proposal — not every conversation leads to one." - ) - else: - thread_phase = "MUST CONCLUDE" - phase_guidance = ( - "This is message 12 — you MUST conclude the thread now. Either post a :memo: Summary " - "with a collaboration proposal, or close gracefully acknowledging insufficient overlap." - ) + # Thread phase guidance + instructions, per role. scout_hub scouts ideas + # against Blackbird's screening rubric; it has no lab and never proposes a + # collaboration. See src/agent/thread_guidance.py. + thread_phase, phase_guidance, instructions = phase4_guidance( + self.role, thread.message_count + ) # Format thread history history_text = "\n".join( f"**{m['sender']}**: {m['content']}" for m in thread_history ) - # Build instructions based on phase - if thread_phase == "EXPLORE": - instructions = ( - "Write a reply that shares specific details from your lab and asks a clarifying " - "question. Use tools proactively to research the other lab." - ) - elif thread_phase == "DECIDE": - instructions = ( - "Write a reply that moves toward a conclusion. Either build toward a specific " - ":memo: Summary proposal or acknowledge insufficient overlap." - ) - else: - instructions = ( - "This is the final message. You MUST either:\n" - "1. Post a :memo: Summary with a specific collaboration proposal, OR\n" - "2. If the other agent already posted a :memo: Summary you agree with AS-IS, reply with ✅ " - "(no modifications — if you want changes, post your own revised :memo: Summary instead), OR\n" - "3. Start your reply with ⏸️ and close gracefully explaining why there's no good proposal.\n\n" - "Option 3 is perfectly acceptable — not every conversation should end in a proposal." - ) - # If the thread's root post is about a paper this lab authored, warn the # model not to engage as if it were external work (see issue #7). root_content = thread_history[0]["content"] if thread_history else "" @@ -514,6 +519,7 @@ def build_phase5_prompt( funding_thread_summaries: dict[str, str] | None = None, visibility: str = VISIBILITY_PUBLIC, channel_id: str | None = None, + post_type_menu: str | None = None, ) -> tuple[str, list[dict]]: """ Build system + messages for Phase 5 new post. @@ -531,10 +537,16 @@ def build_phase5_prompt( always operates in a public channel. The parameters are plumbed through for symmetry with the other phase builders; future work that lets agents initiate private-channel posts will use them. + + post_type_menu: pre-rendered {post_type_menu} block. The engine computes + it from the role's allow-list filtered by the live cohort gate, and + enforces the SAME set when the response comes back. None renders + THIS AGENT'S ROLE's declared set with no topology filtering — used by + direct callers and tests that have no topology to apply. """ system_prompt = self.build_system_prompt(visibility=visibility, channel_id=channel_id) - phase5_template = self._load_file( - PROMPTS_DIR / "phase5-new-post.md", + phase5_template = self._load_prompt( + "phase5-new-post.md", "Choose to reply to an interesting post or make a new top-level post.", ) @@ -633,6 +645,23 @@ def build_phase5_prompt( prompt_text = prompt_text.replace("{subscribed_channels}", channels_text) prompt_text = prompt_text.replace("{your_recent_posts}", recent_text) prompt_text = prompt_text.replace("{prior_conversations}", prior_text) + if post_type_menu is None: + # No topology supplied — render THIS agent's role set with no + # filtering, matching the "gate is None means no filtering" rule. + # Role-aware, not DEFAULT_POST_TYPES: a scout_hub agent built by a + # direct caller would otherwise get the pi_lab menu, offering it + # three types its own role.toml forbids. + # + # gate=None also makes render_menu emit guidance instead of an + # enumeration for an addressed type. There is no roster here to + # enumerate, and the enumeration would come out as the literal + # "one of: ." — in a live prompt, and in test_phase5_prompt_gm's + # committed snapshot. + post_type_menu = render_menu( + load_role(self.role).post_types, gate=None, roles_by_agent={}, + self_id=self.agent_id, bot_names={}, + ) + prompt_text = prompt_text.replace("{post_type_menu}", post_type_menu) # Inject pre-loaded FOA details for Option B (funding collaborations) if thread_foa_contexts: @@ -741,6 +770,16 @@ def _load_file(path: Path, default: str) -> str: return default +# Fallback identity block used only if prompts/identity.md (or a role's +# override) is missing from disk. Must match prompts/identity.md verbatim, +# including the absence of a trailing newline — see _compose_system_prompt, +# which relies on exactly one blank line separating this block from its +# neighbors. +_DEFAULT_IDENTITY = """## Your Identity +You are **{bot_name}**, the AI agent representing the {pi_name} lab at Scripps Research. +Your agent ID is "{agent_id}". When communicating, represent your lab professionally.""" + + def _default_system_prompt() -> str: return """You are an AI agent representing a research lab at Scripps Research in a Slack workspace called "labbot". Your role is to facilitate scientific collaboration by engaging with other lab agents. diff --git a/src/agent/grantbot.py b/src/agent/grantbot.py index 93f41ff..aa17c8a 100644 --- a/src/agent/grantbot.py +++ b/src/agent/grantbot.py @@ -14,7 +14,7 @@ import asyncio import json import logging -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -29,6 +29,8 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from src.agent.ids import WRITER_GRANTBOT, set_default_writer_id +from src.agent.slack_client import SLACK_MAX_TEXT_CHARS, split_for_slack from src.config import get_settings from src.models import GrantbotPostedFoa from src.services.grants import fetch_opportunity_detail, list_posted_opportunities @@ -139,7 +141,7 @@ def _parse_close_date(raw: str) -> datetime | None: return None for fmt in ("%m/%d/%Y", "%Y-%m-%d", "%Y/%m/%d"): try: - return datetime.strptime(raw, fmt).replace(tzinfo=timezone.utc) + return datetime.strptime(raw, fmt).replace(tzinfo=UTC) except ValueError: continue return None @@ -156,6 +158,85 @@ def _has_sufficient_lead_time(close_date_raw: str, now: datetime, min_days: int) return cd >= now + timedelta(days=min_days) +async def _post_funding_to_db(session: AsyncSession, channel_name: str, full_post: str) -> None: + """Write a GrantBot funding post to agent_messages (Slack-off path). + + Authored by the 'grantbot' identity as a top-level post so agents scan it in + Phase 2 and can start funding threads (funding threads are open to all). + """ + from src.agent.ids import mint_local_ts + from src.models import AgentMessage + from src.services.pi_inbox import get_latest_run_id + + run_id = await get_latest_run_id(session) + if not run_id: + raise RuntimeError("No simulation run to post funding opportunity into") + ts = mint_local_ts() + session.add(AgentMessage( + simulation_run_id=run_id, agent_id="grantbot", + channel_id=f"local:{channel_name}", channel_name=channel_name, + message_ts=ts, phase="new_post", visibility="public", + content=full_post, sender_name="GrantBot", is_bot=True, posted_at=float(ts), + )) + await session.flush() + + +async def _post_one_opportunity( + session: AsyncSession, + *, + channel: str, + full_post: str, + opp_num: str, + token: str | None = None, +) -> list[dict[str, Any]]: + """Publish one FOA as N messages, none of them longer than Slack accepts. + + Returns one record per message actually created, ``{"ts", "channel", "text"}``. + + Slack does not reject an over-long ``chat_postMessage``: it splits the body itself and + returns only the *last* message's ts (measured live at >4000 characters). A caller + that posts an unsplit body therefore believes it published one message when the + workspace holds three, and it holds no id for two of them. GrantBot's bodies are + LLM-drafted and prefixed with a header, so their length is not something the call site + controls. Splitting here — via ``slack_web.post_message`` on the Slack branch and + ``split_for_slack`` on the DB branch — makes the count GrantBot reports, the count + Slack holds and the count the mirror stores the same number. + + ``token=None`` is the Slack-off branch: the post goes straight into + ``agent_messages``, one row per chunk, because a single row holding a body Slack would + render as three messages is what breaks the bijection ``split_for_slack`` exists to + keep. On the Slack branch nothing is written here — the simulation's channel poller + already ingests GrantBot's Slack posts keyed by their Slack ts, and a row minted with + a local canonical id would not dedup against it. + """ + if token is None: + chunks = split_for_slack(full_post, SLACK_MAX_TEXT_CHARS) + try: + for chunk in chunks: + await _post_funding_to_db(session, channel, chunk) + except Exception: + # Discard the chunks that did land before re-raising. Splitting opened this + # window: one FOA is now several rows, so a failure can land mid-post, and the + # caller's recovery — ``_release_foa`` — *commits*. Without this the fragment + # becomes permanent and the retry posts the whole FOA on top of it. The claim + # was committed by ``_claim_foa``, so rolling back cannot lose it. + await session.rollback() + raise + logger.info( + "Posted opportunity %s to #%s in %d message(s) (DB)", + opp_num, channel, len(chunks), + ) + return [{"ts": None, "channel": channel, "text": c} for c in chunks] + + from src.services.slack_web import post_message_async + + posted = await post_message_async(token, f"#{channel}", full_post) + logger.info( + "Posted opportunity %s to #%s in %d message(s)", opp_num, channel, len(posted), + ) + return posted + + async def _load_posted_numbers(session: AsyncSession) -> set[str]: """Return the set of already-posted FOA numbers from Postgres.""" result = await session.execute(select(GrantbotPostedFoa.foa_number)) @@ -342,38 +423,40 @@ async def _draft_post( return None -def _ensure_channel_membership(slack_client, channel_names: set[str]) -> None: - """Join any public channels the bot isn't already a member of.""" - try: - # Build a map of channel name -> id for all public channels - channel_map: dict[str, str] = {} - cursor = None - while True: - resp = slack_client.conversations_list( - types="public_channel", - exclude_archived=True, - limit=200, - cursor=cursor, - ) - for ch in resp.get("channels", []): - channel_map[ch["name"]] = ch["id"] - cursor = resp.get("response_metadata", {}).get("next_cursor") - if not cursor: - break +def _ensure_channel_membership(token: str, channel_names: set[str]) -> None: + """Join any public channels the bot isn't already a member of. - for name in channel_names: - clean_name = name.lstrip("#") - ch_id = channel_map.get(clean_name) - if not ch_id: - logger.warning("Channel #%s not found in workspace", clean_name) - continue - try: - slack_client.conversations_join(channel=ch_id) - logger.info("Joined #%s", clean_name) - except Exception as exc: - logger.warning("Could not join #%s: %s", clean_name, exc) + Goes through ``slack_web`` rather than a raw client so the listing is fully paginated + and every call is retried on a 429. The hand-rolled loop this replaces had neither: a + rate-limited ``conversations.list`` raised straight into the ``except`` below, which + logs a warning and returns, and GrantBot then posted to channels it had not joined. + + ``exclude_archived=True`` is deliberate and differs from ``list_channel_ids``'s + default: this map feeds ``conversations_join``, and an archived channel cannot be + joined. Callers that only ask "does this name exist" must count archived channels, + because an archived channel still owns its name — hence the differing default. + """ + from src.services.slack_web import join_channel, list_channel_ids + + try: + channel_map = list_channel_ids( + token, include_private=False, exclude_archived=True + ) except Exception as exc: logger.warning("Failed to list channels for auto-join: %s", exc) + return + + for name in channel_names: + clean_name = name.lstrip("#") + ch_id = channel_map.get(clean_name) + if not ch_id: + logger.warning("Channel #%s not found in workspace", clean_name) + continue + try: + join_channel(token, ch_id) + logger.info("Joined #%s", clean_name) + except Exception as exc: + logger.warning("Could not join #%s: %s", clean_name, exc) async def run_grantbot( @@ -429,7 +512,7 @@ async def _run_grantbot_with_session( # 2b. Drop FOAs with insufficient lead time — labs can't prepare a credible # response for a deadline a few days out. See MIN_LEAD_DAYS. - now = datetime.now(timezone.utc) + now = datetime.now(UTC) short_lead: list[tuple[str, str]] = [] kept: dict[str, dict] = {} for num, opp in all_opps.items(): @@ -456,7 +539,18 @@ async def _run_grantbot_with_session( logger.info("Selected %d opportunities for posting", len(selected_opps)) # 4. Fetch details for selected opportunities + # + # The fallback to the search-shaped `opp` is deliberate — a post with no + # description beats no post. But it used to be SILENT, and that hid an + # upstream outage completely: measured 2026-08-04, grants.gov's detail + # backend answered every id with an outer "Webservice Succeeds" wrapping an + # inner "No response received ... at the backend server", so + # fetch_opportunity_detail returned None for 5/5 real ids. Search hits carry + # no description either, so every drafted post went to the LLM with an empty + # Description field and nothing said so. The tally below is what makes that + # visible rather than indistinguishable from a normal quiet run. detailed_opps = [] + detail_misses = 0 for num, opp in selected_opps.items(): if opp.get("id"): try: @@ -464,10 +558,20 @@ async def _run_grantbot_with_session( if detail: detailed_opps.append(detail) continue + logger.warning("No detail returned for %s (id=%s)", num, opp["id"]) except Exception as exc: - logger.debug("Detail fetch failed for %s: %s", num, exc) + logger.warning("Detail fetch failed for %s: %s", num, exc) + detail_misses += 1 detailed_opps.append(opp) + if detail_misses and detailed_opps: + level = logger.error if detail_misses == len(detailed_opps) else logger.warning + level( + "Grants.gov detail unavailable for %d/%d opportunities — those posts are " + "drafted from title and agency alone, with no description", + detail_misses, len(detailed_opps), + ) + # 4b. Cache FOA details locally for agent access from src.agent.foa_cache import cache_foa for opp in detailed_opps: @@ -497,19 +601,32 @@ async def _run_grantbot_with_session( logger.info("Drafted %d posts, posting %d (max %d per channel)", len(drafted), len(to_post), max_per_channel) - # 6. Post to Slack (or dry-run) + # 6. Post to Slack, or (Slack off) write straight to the DB, or dry-run. posted_list: list[dict] = [] - slack_client = None + # "" means no usable credential, which is a different case from Slack being off: the + # claim has to be released so a later run with a token can still post the FOA. + bot_token = "" + slack_on = False if not dry_run: - from slack_sdk import WebClient - bot_token = getattr(settings, "slack_bot_token_grantbot", "") - if not bot_token or bot_token.startswith("xoxb-placeholder"): - bot_token = settings.slack_bot_token_su - logger.info("No grantbot Slack token — using SuBot's token as fallback") - if bot_token and not bot_token.startswith("xoxb-placeholder"): - slack_client = WebClient(token=bot_token) - _ensure_channel_membership(slack_client, {item.get("channel", channel) for item in to_post}) + from src.services.slack_tokens import slack_globally_enabled + slack_on = await slack_globally_enabled(session) + if slack_on: + candidate = getattr(settings, "slack_bot_token_grantbot", "") + if not candidate or candidate.startswith("xoxb-placeholder"): + candidate = settings.slack_bot_token_su + logger.info("No grantbot Slack token — using SuBot's token as fallback") + if candidate and not candidate.startswith("xoxb-placeholder"): + bot_token = candidate + # to_thread: the helper is sync and makes paginated Slack calls + # with backoff, and this caller is async. Run inline it would hold + # the event loop for the whole listing plus any retry. + await asyncio.to_thread( + _ensure_channel_membership, + bot_token, {item.get("channel", channel) for item in to_post}, + ) + else: + logger.info("Slack disabled — GrantBot posting funding opportunities to the DB") for item in to_post: opp = item["opportunity"] @@ -536,15 +653,32 @@ async def _run_grantbot_with_session( logger.info("Skipping FOA %s — already claimed by another run", opp_num) continue - if not slack_client: - # No Slack client available (no token configured). Release the claim - # so a future run with credentials can post this FOA. + if not slack_on: + # Slack off — write the funding post straight to agent_messages so + # the sim scans it (funding threads are open to all). Keep the claim. + try: + await _post_one_opportunity( + session, channel=target_channel, full_post=full_post, + opp_num=opp_num, + ) + except Exception as exc: + logger.error("Failed to persist %s to #%s: %s", opp_num, target_channel, exc) + await _release_foa(session, opp_num) + continue + posted_list.append({"number": opp_num, "title": title, "channel": target_channel}) + continue + + if not bot_token: + # Slack on but no usable token. Release the claim so a future run with + # credentials can post this FOA. await _release_foa(session, opp_num) continue try: - slack_client.chat_postMessage(channel=f"#{target_channel}", text=full_post) - logger.info("Posted opportunity %s to #%s", opp_num, target_channel) + await _post_one_opportunity( + session, channel=target_channel, full_post=full_post, + opp_num=opp_num, token=bot_token, + ) except Exception as exc: logger.error("Failed to post %s to #%s: %s", opp_num, target_channel, exc) await _release_foa(session, opp_num) @@ -565,7 +699,7 @@ async def _run_grantbot_with_session( def _should_run_today() -> bool: """Return True if grantbot hasn't completed a run today (UTC).""" - today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + today = datetime.now(UTC).strftime("%Y-%m-%d") if LAST_RUN_FILE.exists(): last_date = LAST_RUN_FILE.read_text(encoding="utf-8").strip() return last_date != today @@ -575,7 +709,7 @@ def _should_run_today() -> bool: def _mark_run_complete() -> None: """Record that grantbot ran today.""" LAST_RUN_FILE.parent.mkdir(parents=True, exist_ok=True) - today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + today = datetime.now(UTC).strftime("%Y-%m-%d") LAST_RUN_FILE.write_text(today, encoding="utf-8") @@ -587,6 +721,7 @@ def main( max_per_channel: int = typer.Option(1, "--max-per-channel", help="Max opportunities to post per channel per run"), ): """Search for funding opportunities and post relevant ones to Slack.""" + set_default_writer_id(WRITER_GRANTBOT) results = asyncio.run(run_grantbot( channel=channel, dry_run=dry_run, @@ -618,10 +753,12 @@ def scheduler( """ import time + # Claim this process's canonical-id writer slot before the first post (R1). + set_default_writer_id(WRITER_GRANTBOT) logger.info("GrantBot scheduler started (run_hour=%d UTC, check every %ds)", run_hour, check_interval) while True: - now = datetime.now(timezone.utc) + now = datetime.now(UTC) if _should_run_today() and now.hour >= run_hour: logger.info("Running daily grant search...") try: diff --git a/src/agent/ids.py b/src/agent/ids.py new file mode 100644 index 0000000..b209b31 --- /dev/null +++ b/src/agent/ids.py @@ -0,0 +1,137 @@ +"""Canonical message-id minting: monotonic, unique, ts-shaped ids. + +A *ts-shaped* id is a decimal ``"."`` string, matching +the Slack ts format so the same id column and ``float(ts)`` ordering work whether +a message originated in Slack or was minted locally (Slack-off / DB-origin). + +Why integers: ``float64`` cannot hold microsecond precision at current epoch +magnitudes — the ULP at ~1.75e9 s is ~2.4e-7 s, coarser than the 1e-6 s step the +old ``f"{time.time():.6f}"`` scheme relied on. Two ids minted in the same tick +could therefore format to the *same* 6-decimal string (breaking uniqueness) or +fail to be strictly increasing once round-tripped through a float (breaking the +posted_at ordering). We instead carry a monotonic **integer-microsecond** +high-water mark and only format to a string at the very end, never round-tripping +the fractional part through a float. + +Why a writer slot: a per-process counter cannot stop two *processes* minting the +identical id in the same microsecond, and the DB's only recourse — the +``uq_agent_messages_run_ts`` constraint plus the on-conflict guard — resolves +such a collision by *dropping* one of the two messages. Slack never had this +problem (it issues a globally unique ts) and the DB is now the sole durable +store, so a dropped message is unrecoverable. Each minter therefore owns a +residue class of the microsecond field: ids are quantized to a +``WRITER_SLOT_MODULUS``-microsecond slot and stamped with the writer's id in the +low digits, making a cross-writer collision structurally impossible while +keeping the id ts-shaped, float-parseable and strictly ordered. The cost is +resolution (a writer can mint one id per slot), which is orders of magnitude +above the real posting rate. + +See specs/local-db-conversations.md, the PR #19 review (M1 / mint precision) and +.notes/db-conversations-residual-2026-07-24.md (R1). +""" + +from __future__ import annotations + +import threading +import time + +# Microseconds per slot. The low 2 digits of the 6-digit microsecond field carry +# the writer id, so ids from different writers can never coincide. Keep this a +# power of ten so the ids stay readable and the slot boundary is obvious. +WRITER_SLOT_MODULUS = 100 + +# Writer ids (0 <= id < WRITER_SLOT_MODULUS). Every process/minter that writes a +# canonical id into a shared table needs a distinct one. Both the engine's own +# minter and the module default used *within* the engine process are listed, so +# they can never collide with each other either. +WRITER_ENGINE = 0 # SimulationEngine._ts_minter (agent_messages) +WRITER_WEB = 1 # web app process (PI messages + DMs) +WRITER_GRANTBOT = 2 # grantbot process (funding posts) +WRITER_ENGINE_AUX = 3 # module default inside the engine process (PI DMs) + + +def _fmt(us: int) -> str: + """Format integer microseconds-since-epoch as a ts-shaped id string.""" + return f"{us // 1_000_000}.{us % 1_000_000:06d}" + + +class TsMinter: + """Thread-safe minter of monotonic ts-shaped ids, unique across writers. + + Ids are unique **per process** by the instance's own counter, and unique + **across processes** by ``writer_id``: every id this minter returns is + congruent to ``writer_id`` modulo ``WRITER_SLOT_MODULUS``, a residue class no + other correctly-configured minter uses. The DB's + ``uq_agent_messages_run_ts`` constraint remains the backstop, but it should + no longer be reachable by concurrent minting. + """ + + def __init__(self, writer_id: int = WRITER_ENGINE) -> None: + if not 0 <= writer_id < WRITER_SLOT_MODULUS: + raise ValueError( + f"writer_id must be in [0, {WRITER_SLOT_MODULUS}), got {writer_id}" + ) + self._writer_id = writer_id + self._last_slot = 0 + self._lock = threading.Lock() + + @property + def writer_id(self) -> int: + return self._writer_id + + def seed_floor(self, seconds: float) -> None: + """Raise the high-water mark so subsequent ids sort after ``seconds``. + + Called after a DB rebuild with the max ``posted_at`` seen, so minted ids + always sort after restored history — including history minted by another + writer, whose ids fall in a different residue class. + """ + with self._lock: + self._last_slot = max( + self._last_slot, round(seconds * 1_000_000) // WRITER_SLOT_MODULUS + ) + + def mint(self) -> str: + """Return the next monotonic id in this writer's residue class.""" + with self._lock: + slot = time.time_ns() // 1000 // WRITER_SLOT_MODULUS + # Strictly advance: never reuse a slot, so ids stay unique and + # increasing even when several are minted inside one slot window. + if slot <= self._last_slot: + slot = self._last_slot + 1 + self._last_slot = slot + return _fmt(slot * WRITER_SLOT_MODULUS + self._writer_id) + + +# Process-wide default minter for writers that don't own a SimulationEngine +# instance — the PI web inbox (src/services/pi_inbox.py) and GrantBot +# (src/agent/grantbot.py). Each *process* must claim its writer id at startup +# via set_default_writer_id(); the default below is the web app, the most +# common host for this minter. See WRITER_* above. +_default = TsMinter(WRITER_WEB) + + +def set_default_writer_id(writer_id: int) -> None: + """Claim a writer id for this process's default minter. + + Call at process entry, before anything mints. Swaps in a minter for the new + residue class, carrying over the outgoing minter's high-water mark so ids + stay monotonic across the swap even if something already minted (a fresh + counter could otherwise reuse a slot within the same microsecond window). + """ + global _default + old = _default + new = TsMinter(writer_id) + with old._lock: + new._last_slot = old._last_slot + _default = new + + +def default_writer_id() -> int: + """Return the writer id the process-wide default minter is using.""" + return _default.writer_id + + +def mint_local_ts() -> str: + """Mint a ts-shaped id from the process-wide default minter.""" + return _default.mint() diff --git a/src/agent/main.py b/src/agent/main.py index 1c34ab4..db3c56c 100644 --- a/src/agent/main.py +++ b/src/agent/main.py @@ -10,12 +10,12 @@ import asyncio import logging import signal -import sys from datetime import datetime, timezone import typer from src.agent.agent import Agent +from src.agent.ids import WRITER_ENGINE_AUX, set_default_writer_id from src.agent.simulation import SimulationEngine from src.config import get_settings @@ -31,7 +31,16 @@ @app.command() def main( max_runtime: int = typer.Option(0, "--max-runtime", help="Max runtime in minutes (0 = run until stopped)"), - budget: int = typer.Option(50, "--budget", help="Max LLM calls per agent"), + budget: int = typer.Option( + 0, "--budget", + help=( + "DEPRECATED legacy cumulative cap: max LLM calls per agent for the " + "WHOLE run. 0 (default) disables it. Superseded by the sliding-window " + "rate limiter (llm_calls_per_load_per_window). Passing a nonzero value " + "can permanently bench a hub agent — see " + "docs/specs/2026-08-06-hub-budget-scheduler-design.md §6." + ), + ), mock: bool = typer.Option(False, "--mock", help="Run in mock mode without real Slack tokens"), no_db: bool = typer.Option(False, "--no-db", help="Skip database logging"), fresh: bool = typer.Option(False, "--fresh", help="Wipe simulation data and start fresh"), @@ -39,6 +48,10 @@ def main( all_agents: bool = typer.Option(False, "--all-agents", help="Run every AgentRegistry row regardless of status (default is status='active' only)"), ): """Run the turn-based agent simulation.""" + # Claim this process's canonical-id writer slot before anything mints. The + # engine's own minter owns WRITER_ENGINE; the module default is used here + # only for PI DM rows, so it takes the aux slot (R1). + set_default_writer_id(WRITER_ENGINE_AUX) asyncio.run(_run_simulation(max_runtime, budget, mock, no_db, fresh, reset_cursors, all_agents)) @@ -68,7 +81,7 @@ async def _run_simulation( _sf = _asm(_engine, expire_on_commit=False) async with _sf() as _db: _stmt = _select( - _AR.agent_id, _AR.bot_name, _AR.pi_name, _AR.slack_bot_token + _AR.agent_id, _AR.bot_name, _AR.pi_name, _AR.slack_bot_token, _AR.role ) if not all_agents: _stmt = _stmt.where(_AR.status == "active") @@ -76,7 +89,10 @@ async def _run_simulation( finally: await _engine.dispose() - agents = [Agent(agent_id=r.agent_id, bot_name=r.bot_name, pi_name=r.pi_name) for r in _rows] + agents = [ + Agent(agent_id=r.agent_id, bot_name=r.bot_name, pi_name=r.pi_name, role=r.role) + for r in _rows + ] roster_tokens = {r.agent_id: r.slack_bot_token for r in _rows} if not agents: @@ -91,16 +107,31 @@ async def _run_simulation( len(agents), "all statuses" if all_agents else "status='active'", ) - # Set up Slack clients (Web API only, no Socket Mode). Tokens come from the - # AgentRegistry row, falling back to the legacy .env/config mapping. + # Resolve whether Slack is enabled. --mock forces it off; an explicit + # SLACK_ENABLED env setting wins next; otherwise auto-detect from whether + # any agent has a usable bot token. When off, the DB is the sole store and + # no Slack API calls are made. See specs/local-db-conversations.md. + from src.services.slack_tokens import env_token, is_valid_token + + def _token_for(agent_id: str) -> str | None: + tok = roster_tokens.get(agent_id) + return tok if is_valid_token(tok) else env_token(agent_id) + + if mock: + slack_enabled = False + elif settings.slack_enabled is not None: + slack_enabled = settings.slack_enabled + else: + slack_enabled = any(is_valid_token(_token_for(a.agent_id)) for a in agents) + + # Set up transports. When Slack is on, each agent gets a Web-API client + # (Web API only, no Socket Mode); when off, a NullTransport that no-ops all + # Slack calls so the engine runs identically against the DB. slack_clients = {} - if not mock: + if slack_enabled: from src.agent.slack_client import AgentSlackClient - from src.services.slack_tokens import env_token, is_valid_token for agent in agents: - bot_token = roster_tokens.get(agent.agent_id) - if not is_valid_token(bot_token): - bot_token = env_token(agent.agent_id) + bot_token = _token_for(agent.agent_id) if is_valid_token(bot_token): client = AgentSlackClient( agent_id=agent.agent_id, @@ -112,6 +143,11 @@ async def _run_simulation( logger.warning("[%s] Slack connection failed — skipping", agent.agent_id) else: logger.warning("[%s] No valid Slack token — skipping", agent.agent_id) + else: + from src.agent.transport import NullTransport + for agent in agents: + slack_clients[agent.agent_id] = NullTransport(agent_id=agent.agent_id) + logger.info("Slack disabled — running DB-only (NullTransport for %d agents)", len(agents)) # Set up database session factory session_factory = None @@ -120,7 +156,7 @@ async def _run_simulation( if not no_db: from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - from src.models import AgentChannel, AgentMessage, SimulationRun + from src.models import AgentChannel, AgentMessage, PiDmMessage, SimulationRun engine = create_async_engine(settings.database_url) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) @@ -131,6 +167,7 @@ async def _run_simulation( async with session_factory() as db: await db.execute(AgentMessage.__table__.delete()) await db.execute(AgentChannel.__table__.delete()) + await db.execute(PiDmMessage.__table__.delete()) await db.commit() logger.info("Simulation data wiped.") @@ -195,19 +232,34 @@ async def _run_simulation( session_factory=session_factory, simulation_run_id=simulation_run_id, reset_cursors=reset_cursors, + slack_enabled=slack_enabled, ) # Handle shutdown signals loop = asyncio.get_event_loop() def shutdown(): + # Only flip the stop flag here. The flush must not run in a + # fire-and-forget task: the main loop can return first, and asyncio.run + # then cancels the still-pending task mid-await, losing the in-flight + # turn's messages. It is awaited in the finally-block below instead (R2). logger.info("Received shutdown signal") - asyncio.ensure_future(sim_engine.stop()) + sim_engine.request_stop() for sig in (signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(sig, shutdown) try: + if budget > 0: + logger.warning( + "--budget %d is the DEPRECATED cumulative cap. It counts LLM calls " + "for the ENTIRE run, is rebuilt from llm_call_logs on restart, and " + "therefore benches an agent PERMANENTLY once crossed — this is what " + "took the blackbird hub off the air for 161 consecutive turns. The " + "sliding-window rate limiter supersedes it. Pass --budget 0 unless " + "you specifically want the legacy behaviour.", + budget, + ) logger.info( "Starting simulation: %d agents, %s max runtime, %d budget/agent%s", len(agents), runtime_label, budget, @@ -217,6 +269,15 @@ def shutdown(): except Exception: logger.exception("Simulation engine raised an exception") finally: + # Durably flush buffered messages/LLM logs before anything else. The DB + # is the primary conversation store, so anything still in the in-memory + # buffer at exit is otherwise unrecoverable. Runs on every exit path + # (signal, time limit, budget exhaustion, crash). + try: + await sim_engine.stop() + except Exception: + logger.exception("Final flush on shutdown failed") + # Update simulation run status if session_factory and simulation_run_id: async with session_factory() as db: diff --git a/src/agent/message_log.py b/src/agent/message_log.py index 55c2d25..2887add 100644 --- a/src/agent/message_log.py +++ b/src/agent/message_log.py @@ -2,8 +2,10 @@ import logging import re -from dataclasses import dataclass, field -from typing import Any +from dataclasses import dataclass +from typing import Callable + +from src.visibility import VISIBILITY_COLLAB_PRIVATE logger = logging.getLogger(__name__) @@ -25,6 +27,17 @@ class LogEntry: # memory segment. Default 'public' is safe for all existing callers. # See specs/privacy-and-channel-visibility.md §G2. visibility: str = "public" + # Slack-mirror mapping — set when this message was posted to (or came from) + # Slack. In pure Slack-on mode slack_ts == ts. Persisted to the DB row so the + # reconcile pass can dedup a mirrored message. See specs/local-db-conversations.md. + slack_ts: str | None = None + slack_channel_id: str | None = None + # The *root's* Slack ts for a mirrored reply. Distinct from thread_ts, which + # is the canonical (possibly locally-minted) root id: a thread that started + # Slack-off has a minted root, which is not a valid Slack ts. None means this + # entry has no Slack parent — either it is not a reply, or its thread has no + # Slack presence. See SimulationEngine._slack_parent_ts. + slack_thread_ts: str | None = None def is_funding_post(content: str) -> bool: @@ -32,12 +45,67 @@ def is_funding_post(content: str) -> bool: return ":moneybag:" in content +def _entry_allowed(entry: "LogEntry", allowed_sender_ids: set[str] | None) -> bool: + """Cohort gate for one log entry. See .notes/cohort-system-v2.md §5.1. + + Returns True (entry is visible to the viewing agent) when: + + - ``allowed_sender_ids is None`` — the gate is off for this agent (isolation + disabled, or ``cohort_default_policy="open"`` and the agent is uncohorted); + - the author is a **human** — keyed on ``is_bot``, *not* on + ``sender_agent_id is None``. ``agent_messages.agent_id`` is nullable, so a + bot-authored row written with a NULL agent_id ingests through + ``_poll_inbound_from_db`` as ``sender_agent_id=None`` and would otherwise + pass the gate as a human; + - the entry is in a ``collab_private`` channel — a PI explicitly paired those + two agents via the reopen flow, and an admin-level grouping must not veto an + explicit human pairing. Read from the persisted ``LogEntry.visibility`` + rather than the engine's in-memory channel map, so it is correct for rows + ingested from another process and after a restart; + - the author shares at least one cohort with the viewing agent. + + Named ``_entry_allowed``, not ``_sender_allowed``: it is no longer a function + of the sender alone. + """ + if allowed_sender_ids is None: + return True + if not entry.is_bot: + return True + if entry.visibility == VISIBILITY_COLLAB_PRIVATE: + return True + if entry.sender_agent_id is None: + # A bot row with no agent_id cannot be attributed to a cohort. Fail closed: + # unattributable bot traffic must not leak through the human bypass. + return False + return entry.sender_agent_id in allowed_sender_ids + + class MessageLog: """ Append-only in-memory message log. All posts and replies are recorded here. Agents query it to find new posts since their last turn, thread histories, etc. + + **Cohort-gate classification (.notes/cohort-system-v2.md §6).** Every public + read method is classified GATED or UNGATED below, and the classification is + repeated in each method's docstring. ``tests/unit/test_cohort_isolation.py`` + fails if a new public ``get_*``/``has_*`` method appears without one, so the + inventory cannot silently rot. + + GATED — takes ``allowed_sender_ids`` and drops entries the viewing agent may + not act on: + get_new_top_level_posts, get_replies_to_agent_posts, get_tags_for_agent, + has_new_reply_from_other + + UNGATED by design — thread-internal, self-authored, or bookkeeping: + get_entry, get_thread_history, get_thread_message_count, + get_agent_top_level_posts, get_last_bot_sender_in_channel, + get_thread_allowed_agents, is_funding_thread, latest_timestamp + + Writes (``append`` / ``load_entry`` / ``_record``) are NEVER gated: the log is + shared by every agent in the process, so filtering at ingest would filter for + all of them at once. The gate belongs at the per-agent read. See v2 §6.2. """ def __init__(self) -> None: @@ -45,18 +113,69 @@ def __init__(self) -> None: self._by_ts: dict[str, LogEntry] = {} # ts -> entry for fast lookup # Map bot_name (lowercase) -> agent_id, set by SimulationEngine self._bot_name_to_id: dict[str, str] = {} + # Optional persistence hook, invoked once per *new* append. The engine + # registers this to mirror the log into the DB (the primary store). + # Kept as a plain callback so this module stays DB-agnostic. See + # specs/local-db-conversations.md. + self._persist_cb: Callable[[LogEntry], None] | None = None + # High-water mark over posted_at, maintained on every add. Insertion + # order is NOT time order (see _record), so latest_timestamp cannot read + # the tail of _entries. + self._max_posted_at: float = 0.0 def set_bot_name_map(self, mapping: dict[str, str]) -> None: """Register bot_name -> agent_id mapping (lowercase keys).""" self._bot_name_to_id = dict(mapping) - def append(self, entry: LogEntry) -> None: - """Add a message to the log.""" + def set_persist_callback(self, cb: Callable[[LogEntry], None] | None) -> None: + """Register a callback fired after each new append (for DB persistence).""" + self._persist_cb = cb + + def append(self, entry: LogEntry) -> bool: + """Add a message to the log. + + Idempotent: if an entry with this ts is already present, the append is + skipped (both the in-memory add and the persist callback) and False is + returned. This unifies the previously scattered ``get_entry`` guards and + keeps the DB persist hook from double-writing during Slack reconciliation. + Safe because ids are unique (Slack ts or a minted ts; see mint_ts). + Returns True when a new entry was added. + """ + if entry.ts in self._by_ts: + return False + self._record(entry) + if self._persist_cb is not None: + self._persist_cb(entry) + return True + + def load_entry(self, entry: LogEntry) -> None: + """Append a restored entry WITHOUT firing the persist callback. + + Used by the DB-rebuild path so rows just read from the DB are not + re-persisted. Still idempotent on ts. + """ + if entry.ts in self._by_ts: + return + self._record(entry) + + def _record(self, entry: LogEntry) -> None: + """Store an entry and advance the posted_at high-water mark. + + The log is append-only in *insertion* order, which is not time order: the + DB inbound poller and the Slack reconcile append entries whose posted_at + can predate messages already stored. Every "most recent" query therefore + has to key on posted_at rather than on the tail of ``_entries``. + """ self._entries.append(entry) self._by_ts[entry.ts] = entry + if entry.posted_at > self._max_posted_at: + self._max_posted_at = entry.posted_at def get_entry(self, ts: str) -> LogEntry | None: - """Look up a single entry by its timestamp.""" + """Look up a single entry by its timestamp. + + COHORT-GATE: UNGATED — single-id lookup; callers already know the id. + """ return self._by_ts.get(ts) def get_new_top_level_posts( @@ -64,10 +183,15 @@ def get_new_top_level_posts( since: float, channels: set[str], exclude_agent_id: str, + allowed_sender_ids: set[str] | None = None, ) -> list[LogEntry]: """ Return top-level posts (thread_ts is None) in the given channels, posted after `since`, excluding posts from `exclude_agent_id`. + + When `allowed_sender_ids` is provided, only posts from those agents (plus + human PI posts) are returned — the cohort gate (see specs/cohort-system.md). + COHORT-GATE: GATED via allowed_sender_ids. """ results = [] for entry in self._entries: @@ -79,13 +203,32 @@ def get_new_top_level_posts( continue if entry.sender_agent_id == exclude_agent_id: continue + if not _entry_allowed(entry, allowed_sender_ids): + continue results.append(entry) return results def get_thread_history(self, thread_ts: str) -> list[LogEntry]: - """Return all messages in a thread (including the root post), ordered by time.""" + """Return all messages in a thread (including the root post), ordered by time. + + Ordered by ``posted_at``, not by log-insertion order. Appends from a + single process arrive roughly in time order, but the DB inbound poller + and the Slack reconcile append entries whose posted_at can predate + messages already in the log — so insertion order would hand the LLM a + subtly scrambled thread, unlike the Slack-primary rebuild which fetched + in ts order. The sort is stable, so entries sharing a posted_at keep + their insertion order. The root is pinned first regardless: it is the + thread's parent by definition, even if a reply carries an earlier + posted_at (a writer's clock can run behind — see PI_INBOX_LOOKBACK_S). + + COHORT-GATE: UNGATED by design — once a thread is open its full history + is context, including a partner who has since left the cohort (v2 §8). + """ root = self._by_ts.get(thread_ts) - replies = [e for e in self._entries if e.thread_ts == thread_ts] + replies = sorted( + (e for e in self._entries if e.thread_ts == thread_ts), + key=lambda e: e.posted_at, + ) result = [] if root: result.append(root) @@ -93,17 +236,30 @@ def get_thread_history(self, thread_ts: str) -> list[LogEntry]: return result def get_thread_message_count(self, thread_ts: str) -> int: - """Count total messages in a thread (root + replies).""" + """Count total messages in a thread (root + replies). + + COHORT-GATE: UNGATED by design — bookkeeping over one thread. + """ count = 1 if thread_ts in self._by_ts else 0 count += sum(1 for e in self._entries if e.thread_ts == thread_ts) return count def get_agent_top_level_posts(self, agent_id: str, limit: int = 10) -> list[LogEntry]: - """Return the agent's own top-level posts, most recent first.""" - posts = [ - e for e in self._entries - if e.sender_agent_id == agent_id and e.thread_ts is None - ] + """Return the agent's ``limit`` newest top-level posts, oldest first. + + "Newest" is by ``posted_at``, not by position in the log: a late append of + older history (DB poll / Slack reconcile — see _record) would otherwise + push a genuinely recent post out of the slice, which silently weakens both + callers — the Phase 5 dedup context and the daily post cap. + COHORT-GATE: UNGATED by design — the agent's own posts. + """ + posts = sorted( + ( + e for e in self._entries + if e.sender_agent_id == agent_id and e.thread_ts is None + ), + key=lambda e: e.posted_at, + ) return posts[-limit:] def get_last_bot_sender_in_channel(self, channel_name: str) -> str | None: @@ -112,24 +268,37 @@ def get_last_bot_sender_in_channel(self, channel_name: str) -> str | None: Returns None if no bot has posted there yet. Used to enforce turn-taking in flat collab_private channels (a bot shouldn't post back-to-back there without the other bot responding first). + + "Most recent" is by ``posted_at``. Scanning ``reversed(_entries)`` instead + would let a late-appended *older* message answer as the last poster and + hand the turn to the wrong bot. Ties keep the later insertion, matching + the previous behaviour when posted_at values collide. + COHORT-GATE: UNGATED by design — turn-taking within one channel, and the + only callers are collab_private channels, which the gate exempts (v2 §7). """ - for entry in reversed(self._entries): + best: LogEntry | None = None + for entry in self._entries: if entry.channel != channel_name: continue - if not entry.is_bot: + if not entry.is_bot or not entry.sender_agent_id: continue - if entry.sender_agent_id: - return entry.sender_agent_id - return None + if best is None or entry.posted_at >= best.posted_at: + best = entry + return best.sender_agent_id if best else None def get_replies_to_agent_posts( self, agent_id: str, since: float, + allowed_sender_ids: set[str] | None = None, ) -> list[LogEntry]: """ Find replies (since cursor) to top-level posts authored by agent_id, where the reply is from a different agent. + + When `allowed_sender_ids` is provided, replies from non-cohort agents are + excluded (the cohort gate; human PI replies always pass). + COHORT-GATE: GATED via allowed_sender_ids. """ # First, find all top-level posts by this agent agent_post_ts = { @@ -144,6 +313,8 @@ def get_replies_to_agent_posts( continue if entry.sender_agent_id == agent_id: continue + if not _entry_allowed(entry, allowed_sender_ids): + continue results.append(entry) return results @@ -151,16 +322,23 @@ def get_tags_for_agent( self, agent_bot_name: str, since: float, + allowed_sender_ids: set[str] | None = None, ) -> list[LogEntry]: """ Find posts/replies that mention (tag) the given agent bot name, posted since the given cursor. + + When `allowed_sender_ids` is provided, tags authored by non-cohort agents + are excluded (the cohort gate; human PI tags always pass). + COHORT-GATE: GATED via allowed_sender_ids. """ tag = f"@{agent_bot_name}".lower() results = [] for entry in self._entries: if entry.posted_at <= since: continue + if not _entry_allowed(entry, allowed_sender_ids): + continue if tag in entry.content.lower(): results.append(entry) return results @@ -175,6 +353,7 @@ def get_thread_allowed_agents(self, thread_ts: str) -> set[str] | None: - If no tag, falls back to generic 2-party rule: the first two distinct agents to post are the only allowed participants. - Returns None if the thread root is not found. + COHORT-GATE: UNGATED by design — thread participation rules, not cohort. """ root = self._by_ts.get(thread_ts) if not root: @@ -208,7 +387,10 @@ def get_thread_allowed_agents(self, thread_ts: str) -> set[str] | None: return set(participants) def is_funding_thread(self, thread_ts: str) -> bool: - """Return True if the thread root is a funding post.""" + """Return True if the thread root is a funding post. + + COHORT-GATE: UNGATED by design — a property of the thread root. + """ root = self._by_ts.get(thread_ts) return bool(root and is_funding_post(root.content)) @@ -225,23 +407,42 @@ def has_new_reply_from_other( thread_ts: str, agent_id: str, since: float, + allowed_sender_ids: set[str] | None = None, ) -> bool: - """Check if the other participant posted a new reply since `since`.""" + """Check if the other participant posted a new reply since `since`. + + COHORT-GATE: GATED via allowed_sender_ids. + + See .notes/cohort-system-v2.md §6, §8. This is the read that drives + both the reactive-priority tier (``_owes_reply``) and the Phase 4 reply + decision, so leaving it ungated made the scheduler prioritise exactly the + threads the gate had rejected. Callers pass ``allowed_sender_ids=None`` for + a thread that is already open and not grandfathered — an open conversation + is entitled to conclude (v2 §8) — and pass the agent's gate otherwise. + """ for entry in self._entries: if entry.thread_ts != thread_ts: continue if entry.posted_at <= since: continue - if entry.sender_agent_id != agent_id: - return True + if entry.sender_agent_id == agent_id: + continue + if not _entry_allowed(entry, allowed_sender_ids): + continue + return True return False @property def latest_timestamp(self) -> float: - """Return the timestamp of the most recent entry, or 0.""" - if not self._entries: - return 0.0 - return self._entries[-1].posted_at + """Return the highest posted_at in the log, or 0.0 when it is empty. + + The maximum, not ``_entries[-1].posted_at``: the last-inserted entry is + not the newest one whenever the DB poller or the Slack reconcile has + appended older history (see _record). A cursor taken from the tail could + therefore move *backwards*. + COHORT-GATE: UNGATED by design — global high-water mark for cursors. + """ + return self._max_posted_at def __len__(self) -> int: return len(self._entries) diff --git a/src/agent/pi_handler.py b/src/agent/pi_handler.py index f4ec7cf..a27c394 100644 --- a/src/agent/pi_handler.py +++ b/src/agent/pi_handler.py @@ -28,6 +28,7 @@ def __init__( pi_slack_id_to_agent_ids: dict[str, list[str]], message_log: MessageLog, session_factory=None, + simulation_run_id=None, ): self.agents = agents self.slack_clients = slack_clients @@ -40,6 +41,7 @@ def __init__( } self.message_log = message_log self.session_factory = session_factory + self.simulation_run_id = simulation_run_id # ------------------------------------------------------------------ # DM handling @@ -81,6 +83,10 @@ async def _classify_dm(self, text: str) -> dict[str, Any]: max_tokens=200, log_meta={"agent_id": "pi_handler", "phase": "dm_classify"}, ) + # Deliberately NOT recorded against any Agent: this row is logged + # under the synthetic agent_id "pi_handler", so the restart rebuild + # attributes it to nobody. Counting it live would make the in-process + # ledger disagree with the rebuilt one in the other direction. return self._parse_json(response) except Exception as exc: logger.warning("DM classification failed: %s", exc) @@ -110,7 +116,20 @@ async def _handle_standing_instruction( model=settings.llm_agent_model_sonnet, max_tokens=2000, log_meta={"agent_id": agent_id, "phase": "profile_rewrite"}, + # Fires immediately if llm.py's max_tokens retry actually makes + # a second call, so a retried turn still books as one call per + # real API call, not one per turn. See Agent.record_api_call + # and the explicit record_api_call() just below, which books + # the (at least one) call every turn makes regardless. + on_retry=agent.record_api_call, ) + # This call is logged to llm_call_logs under a REAL agent_id, so the + # restart rebuild (simulation step 4/4b) will attribute it to this + # agent. Recording it here is what keeps the live counters and the + # rebuilt ones consistent — without it, a PI DM burst is invisible to + # the rate limiter now and throttles the agent from turn 0 after a + # restart. See Agent.record_api_call. + agent.record_api_call() # Parse profile and changes from response profile_match = re.search(r"(.*?)", response, re.DOTALL) @@ -200,7 +219,11 @@ async def _handle_question(self, agent_id: str, pi_slack_id: str, question: str) model=settings.llm_agent_model_sonnet, max_tokens=800, log_meta={"agent_id": agent_id, "phase": "pi_question"}, + on_retry=agent.record_api_call, ) + # Logged under a real agent_id -> counted by the restart rebuild, so + # it must be counted live too. See _handle_standing_instruction. + agent.record_api_call() await self._send_dm(agent_id, pi_slack_id, response.strip()) except Exception as exc: logger.error("[%s] Failed to answer PI question: %s", agent_id, exc) @@ -369,12 +392,33 @@ async def notify_thread_conclusion( # ------------------------------------------------------------------ async def _send_dm(self, agent_id: str, pi_slack_id: str, text: str) -> None: - """Send a DM from the agent's bot to the PI.""" + """Send a DM from the agent's bot to the PI (Slack + DB record). + + Persists an outbound row so the DM is durable and visible in the web UI + even when Slack is off. See specs/local-db-conversations.md. + """ client = self.slack_clients.get(agent_id) + slack_ts = None if client and client.is_connected: - client.send_dm(pi_slack_id, text) + result = client.send_dm(pi_slack_id, text) + if isinstance(result, dict): + slack_ts = result.get("ts") else: - logger.debug("[%s] Cannot send DM — no connected client", agent_id) + logger.debug("[%s] Cannot send DM via Slack — recording to DB only", agent_id) + + if self.session_factory and self.simulation_run_id: + try: + from src.services.pi_inbox import record_pi_dm + agent = self.agents.get(agent_id) + async with self.session_factory() as db: + await record_pi_dm( + db, run_id=self.simulation_run_id, agent_id=agent_id, + pi_user_id=pi_slack_id, direction="outbound", content=text, + sender_name=agent.bot_name if agent else agent_id, slack_ts=slack_ts, + ) + await db.commit() + except Exception as exc: + logger.debug("[%s] Could not record outbound DM: %s", agent_id, exc) @staticmethod def _parse_json(text: str) -> dict: diff --git a/src/agent/post_types.py b/src/agent/post_types.py new file mode 100644 index 0000000..b6561b5 --- /dev/null +++ b/src/agent/post_types.py @@ -0,0 +1,324 @@ +"""The post-type vocabulary, and the role + topology filter over it. + +A "post type" is what an agent may emit as a NEW top-level post +(``action: "new_post"`` in the phase-5 response). ``action: "reply"`` is not +governed here. + +Dependency-free on purpose (no src.models, no DB, no Agent import), like +src/agent/roles.py and src/agent/thread_guidance.py, so the filter is +unit-testable without a database, an engine, or a running loop. + +Why the filter exists: the phase-5 prompt used to tell every agent to tag a peer +lab, while the cohort gate forbade every such tag. Measured over one run, 259 +:bulb: Idea posts produced 2 replies (0.8%) and 146 of 146 tagged posts addressed +an agent the poster could not reach. Role alone is the wrong axis — ``pi_lab`` in +a mesh deployment *should* make cross-lab posts; the same role in a star must not. +So a post type declares the counterparty roles it addresses, and availability is +computed against the agent's live gate. See +docs/specs/2026-08-06-role-topology-post-type-gating-design.md. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class PostTypeSpec: + """One post type. ``targets`` is the set of ``AgentRegistry.role`` values this + type addresses; empty means it addresses no one (a broadcast).""" + + name: str + emoji: str + label: str + when_to_use: str + targets: frozenset[str] = frozenset() + + +# Every type the codebase knows. A role may only declare names from this table — +# an unknown name is dropped with a WARNING rather than silently inventing a type +# the prompt has no instructions for. +CANONICAL: dict[str, PostTypeSpec] = { + s.name: s + for s in ( + PostTypeSpec( + "paper", ":newspaper:", "Paper", + "Share a recent publication with a specific finding others could build on.", + ), + PostTypeSpec( + "help_wanted", ":sos:", "Help Wanted", + "Seek a specific capability, reagent, dataset, or expertise your lab " + "genuinely needs and cannot produce in-house.", + ), + PostTypeSpec( + "introduction", ":wave:", "Introduction", + "Introduce your lab's interests and expertise. Use sparingly — only if " + "you have not introduced yourself in this channel yet.", + ), + PostTypeSpec( + "idea_crosslab", ":bulb:", "Idea (cross-lab)", + "Propose an idea at the interface between your lab and another specific " + "lab. Name a concrete first experiment or dataset exchange.", + targets=frozenset({"pi_lab"}), + ), + PostTypeSpec( + "pitch", ":bulb:", "Pitch to the scouting hub", + "Offer one of your OWN lab's ideas for screening — something that might " + "be patentable, fundable, or commercializable. Not a collaboration " + "proposal, and never a suggestion that two other labs should talk.", + targets=frozenset({"scout_hub"}), + ), + PostTypeSpec( + "funding_collab", ":moneybag:", "Funding collaboration", + "Start a funding-originated collaboration around a specific FOA. Must " + "include the FOA number.", + targets=frozenset({"pi_lab"}), + ), + PostTypeSpec( + "opportunity_assessment", ":mag:", "Opportunity Assessment", + "The completed screening artifact for Blackbird staff and the PI.", + ), + ) +} + +# ``pi_lab`` has no role.toml — "pi_lab is the absence of overrides" (roles.py). +# So this tuple IS pi_lab's declared list. Explicit rather than "everything in +# CANONICAL", for the same reason roles.DEFAULT_TOOLS is: adding a new type must +# never silently hand it to every role. +DEFAULT_POST_TYPES: tuple[PostTypeSpec, ...] = ( + CANONICAL["paper"], + CANONICAL["help_wanted"], + CANONICAL["introduction"], + CANONICAL["idea_crosslab"], + CANONICAL["pitch"], + CANONICAL["funding_collab"], +) + +# Types that count as funding actions. In funding_only mode (the agent is blocked +# for regular posts) the available set is narrowed to these. +FUNDING_POST_TYPES: frozenset[str] = frozenset({"funding_collab"}) + +# Retired names a running deployment may still emit. ``idea`` sat in the old +# phase-5 enum alongside ``idea_crosslab`` with no documented difference and no +# code distinguishing them (design §2), so collapsing them is right — but a mesh +# deployment whose bind-mounted prompts lag the baked-in code would otherwise +# have every ``idea`` post rejected by layer 1 and silently publish nothing. +# That is a regression in a deployment this change is not supposed to touch. +# +# Aliases resolve on INPUT only. They are deliberately absent from CANONICAL, +# from any role's declared list, and from every rendered menu, so nothing here +# re-offers a name the vocabulary retired. +LEGACY_POST_TYPE_ALIASES: dict[str, str] = {"idea": "idea_crosslab"} + + +def resolve_post_type_name(name: str) -> str: + """Map a retired post-type name onto its current one; pass anything else through.""" + return LEGACY_POST_TYPE_ALIASES.get(name, name) + + +# Roles a `targets` entry may name. Kept here rather than imported from roles.py +# to avoid a cycle; roles.available_roles() is filesystem-derived and would make +# this module depend on the prompts directory. +_KNOWN_ROLES: frozenset[str] = frozenset({"pi_lab", "scout_hub"}) + + +def parse_post_types(raw: object, *, role: str) -> tuple[PostTypeSpec, ...]: + """Parse the ``post_types`` key of a role manifest. Never raises. + + ``None`` (key absent) yields ``DEFAULT_POST_TYPES``. Anything that is not a + list yields the defaults with a WARNING. Individual malformed or unknown + entries are dropped with a WARNING and the rest are kept — the same + degradation roles.load_role uses for ``tools``. + """ + if raw is None: + return DEFAULT_POST_TYPES + if not isinstance(raw, list): + logger.warning( + "[post_types] %s: post_types must be a list of tables, got %s — " + "using defaults", role, type(raw).__name__, + ) + return DEFAULT_POST_TYPES + + # A dict, not a list: a later `[[post_types]]` entry for a name already + # seen replaces the earlier one (last wins) rather than appending a second, + # contradictory line to the rendered menu. Re-assigning an existing key does + # not move it, so declaration order is still the position of the FIRST + # occurrence of each name — stable between turns. + kept: dict[str, PostTypeSpec] = {} + for entry in raw: + if not isinstance(entry, dict): + logger.warning( + "[post_types] %s: post_types entry is not a table (%r) — dropped", + role, entry, + ) + continue + name = entry.get("name") + if not isinstance(name, str) or not name: + logger.warning( + "[post_types] %s: post_types entry has no usable name (%r) — dropped", + role, entry, + ) + continue + base = CANONICAL.get(name) + if base is None: + logger.warning( + "[post_types] %s: unknown post type %r in role.toml — dropped", + role, name, + ) + continue + targets = base.targets + declared = entry.get("targets") + if declared is not None: + if not isinstance(declared, list) or not all( + isinstance(x, str) for x in declared + ): + logger.warning( + "[post_types] %s: %s targets must be a list of role names, " + "got %r — keeping the canonical default %s", + role, name, declared, sorted(base.targets), + ) + else: + targets = frozenset(declared) + unknown = targets - _KNOWN_ROLES + if unknown: + logger.warning( + "[post_types] %s: %s targets name unknown role(s) %s — the " + "type will never be offered", + role, name, sorted(unknown), + ) + if base.name in kept: + logger.warning( + "[post_types] %s: duplicate post_types entry for %r — the " + "later one wins", + role, base.name, + ) + kept[base.name] = PostTypeSpec( + name=base.name, emoji=base.emoji, label=base.label, + when_to_use=base.when_to_use, targets=targets, + ) + return tuple(kept.values()) + + +def eligible_targets( + spec: PostTypeSpec, + *, + gate: set[str] | None, + roles_by_agent: dict[str, str], + self_id: str, +) -> frozenset[str]: + """Agents this post type may address, given the acting agent's gate. + + Self is always excluded — an agent's own role sits in its own gate, and an + agent is never its own counterparty. An agent with no known role (e.g. + ``grantbot``, which has cohort memberships but no ``AgentRegistry`` row) + matches no ``targets``. + + ``gate is None`` means the cohort gate is off for this agent, so every agent + with a matching role is reachable. + """ + if not spec.targets: + return frozenset() + candidates = roles_by_agent if gate is None else { + aid: r for aid, r in roles_by_agent.items() if aid in gate + } + return frozenset( + aid for aid, r in candidates.items() + if aid != self_id and r in spec.targets + ) + + +def available_for( + declared: tuple[PostTypeSpec, ...], + *, + gate: set[str] | None, + roles_by_agent: dict[str, str], + self_id: str, + funding_only: bool, +) -> tuple[PostTypeSpec, ...]: + """The post types this agent may use as a new top-level post, right now. + + Declaration order is preserved so the rendered menu is stable between turns. + A type with no ``targets`` is always available. A type with ``targets`` is + available only when at least one reachable agent has a matching role. + + ``funding_only`` narrows the result to ``FUNDING_POST_TYPES``; the result may + legitimately be empty in that mode, which must NOT be treated as "skip the + turn" — a funding *reply* is still valid. See spec §5. + """ + out = [ + s for s in declared + if not s.targets + or eligible_targets( + s, gate=gate, roles_by_agent=roles_by_agent, self_id=self_id + ) + ] + if funding_only: + out = [s for s in out if s.name in FUNDING_POST_TYPES] + return tuple(out) + + +_EMPTY_MENU = ( + "**No new top-level post type is available to you this turn.** Do not use " + "`action: \"new_post\"` — it will be rejected and nothing will be posted. " + "Reply to an existing post (Option A) or skip (Option D)." +) + + +def render_menu( + specs: list[PostTypeSpec] | tuple[PostTypeSpec, ...], + *, + gate: set[str] | None, + roles_by_agent: dict[str, str], + self_id: str, + bot_names: dict[str, str], +) -> str: + """Render the available set as the prompt's ``{post_type_menu}``. + + Never returns an empty string, and never prints an empty enumeration. An + addressed type has two renderings: + + - **gate set** — enumerate the reachable agents, because the list is short + and naming them is the whole point. + - **gate None** — guidance only. A mesh has ~50 reachable labs; enumerating + them into every phase-5 prompt would recreate the lab directory this + design is shrinking. It is also the path a caller with no topology takes + (``build_phase5_prompt`` with no menu), where the enumeration would come + out as the literal ``one of: .`` and land in a snapshot. + """ + if not specs: + return _EMPTY_MENU + lines: list[str] = [] + for s in specs: + head = f"- **`{s.name}`** — {s.emoji} {s.label}. {s.when_to_use}" + if not s.targets: + lines.append(head + " Addresses no one — do not tag anyone; set " + "`tagged_agent` to `null`.") + continue + if gate is None: + roles = " or ".join(sorted(s.targets)) + lines.append( + head + f" Addresses one agent whose role is {roles} — set " + "`tagged_agent` to that agent's `agent_id` and tag its @BotName " + "in the message body." + ) + continue + reachable = sorted( + eligible_targets( + s, gate=gate, roles_by_agent=roles_by_agent, self_id=self_id + ) + ) + if not reachable: + # available_for already drops these, so reaching here means the + # caller passed an unfiltered list. Drop it rather than printing + # "Set tagged_agent to exactly one of: ." — offering a type with an + # empty target list is worse than not offering it. + continue + named = ", ".join(f"`{aid}` (@{bot_names.get(aid, aid + 'Bot')})" for aid in reachable) + lines.append( + head + f" Set `tagged_agent` to exactly one of: {named}. " + "Tagging anyone else gets the post rejected." + ) + return "\n".join(lines) if lines else _EMPTY_MENU diff --git a/src/agent/roles.py b/src/agent/roles.py new file mode 100644 index 0000000..cb5d147 --- /dev/null +++ b/src/agent/roles.py @@ -0,0 +1,124 @@ +"""Per-role agent customization: prompt-path resolution and role manifests. + +Dependency-free on purpose (no src.models, no DB) so the resolution rules are +unit-testable without a database, and so src/agent/agent.py can import it +without pulling the ORM into the Agent class. See +docs/specs/2026-08-05-hub-bot-customization-design.md. +""" + +from __future__ import annotations + +import logging +import tomllib +from dataclasses import dataclass +from pathlib import Path + +from src.agent.post_types import DEFAULT_POST_TYPES, PostTypeSpec, parse_post_types + +logger = logging.getLogger(__name__) + +PROMPTS_DIR = Path("prompts") +ROLES_DIR = PROMPTS_DIR / "roles" +DEFAULT_ROLE = "pi_lab" + +# Explicit, NOT "every tool in TOOL_DEFINITIONS": if the default were "all tools", +# adding a new tool to that list would silently hand it to every agent. Explicit +# default keeps every new tool opt-in. See design §4.1. +DEFAULT_TOOLS: frozenset[str] = frozenset( + {"retrieve_profile", "retrieve_abstract", "retrieve_full_text", "retrieve_foa"} +) + + +@dataclass(frozen=True) +class RoleSpec: + name: str + label: str + tools: frozenset[str] + # Optional per-role override for Settings.llm_calls_per_load_per_window. + # None means "use the global setting". This exists to pin a specific agent; + # it is NOT the mechanism — the load signal is (design §4.4). No role sets it. + calls_per_load_per_window: int | None = None + # Layer 1 of post-type gating: what this role may emit as a NEW top-level + # post. Defaults to DEFAULT_POST_TYPES, which IS the pi_lab set (pi_lab has + # no role.toml — the absence of overrides is pi_lab). + post_types: tuple[PostTypeSpec, ...] = DEFAULT_POST_TYPES + + +def available_roles() -> list[str]: + """Every role an agent can be assigned: ``pi_lab`` plus every directory under + ``prompts/roles/``. ``pi_lab`` is listed even if its directory does not exist + (it is the absence of overrides, see ``resolve_prompt_path``) and always comes + first so callers (e.g. the admin ` was built from, so a stale or hand-crafted form can never + write a role the runtime does not know how to resolve. + """ + result = await db.execute( + select(AgentRegistry).where(AgentRegistry.id == agent_id) + ) + agent = result.scalar_one_or_none() + if not agent: + raise HTTPException(status_code=404, detail="Agent not found") + + if role not in available_roles(): + return RedirectResponse( + url=f"/admin/agents/{agent_id}?role_error=Unknown+role", status_code=302 + ) + + agent.role = role + await db.commit() + + return RedirectResponse(url=f"/admin/agents/{agent_id}", status_code=302) + + @router.post("/impersonate") async def impersonate_user( request: Request, @@ -1296,6 +1366,537 @@ async def admin_waitlist_mark_contacted( ) signup = result.scalar_one_or_none() if signup: - signup.contacted_at = datetime.now(timezone.utc) + signup.contacted_at = datetime.now(UTC) await db.commit() return RedirectResponse(url="/admin/waitlist", status_code=302) + + +# --------------------------------------------------------------------------- +# Cohorts — admin-managed groups gating which agents interact during simulation. +# +# The gate is an agent-BEHAVIOUR filter, never access control: it changes what an +# agent acts on, never what a human can read. Nothing in this section may be reused +# to scope a PI-facing view. See .notes/cohort-system-v2.md §6.2. +# +# Enforcement only happens in the running simulation, and only when +# settings.cohort_isolation_enabled is True. Membership edits are picked up live on +# the engine's roster-sync cadence (~30s) — no restart. Filtering is forward-only: +# adding an agent to a cohort does not reveal the backlog it missed while excluded, +# because the agent's cursor has already advanced past it (v2 §6.3). +# --------------------------------------------------------------------------- + +# Cohort name: lowercase alphanumeric + hyphens, max 48 chars (slug style). +_COHORT_NAME_RE = re.compile(r"^[a-z0-9-]{1,48}$") + +# Starlette's request.form() defaults to max_fields=1000. The topology matrix +# posts one marker per rendered row and column plus one value per ticked cell, +# so the payload is agents + cohorts + ticked — but "ticked" alone will pass +# 1,000 on a large enough roster, so the limit is raised rather than relied on. +_TOPOLOGY_MAX_FIELDS = 50_000 + + +async def _cohort_gate_context(db: AsyncSession) -> dict[str, Any]: + """Preview of the gate the engine will compute from the current topology. + + Uses the same ``compute_gates`` the engine uses, so the preview cannot drift from + the behaviour. The roster is AgentRegistry's *active* agents — what the engine + loads — so an inactive agent shows as absent rather than as unrestricted. + See v2 §12. + """ + settings = get_settings() + active = (await db.execute( + select(AgentRegistry.agent_id, AgentRegistry.bot_name) + .where(AgentRegistry.status == "active") + .order_by(AgentRegistry.bot_name) + )).all() + agent_ids = [r.agent_id for r in active] + rows = (await db.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + cohort_count = (await db.execute( + select(func.count()).select_from(Cohort) + )).scalar() or 0 + + gates, preflight_error = compute_gates( + membership_rows=[(r[0], r[1]) for r in rows], + agent_ids=agent_ids, + isolation_enabled=settings.cohort_isolation_enabled, + policy=settings.cohort_default_policy, + cohort_count=cohort_count, + has_db=True, + ) + + # Most recent topology snapshot written by a running engine — the only way this + # process can see the engine's in-memory counters (v2 §9.4 / §13.1). + snapshot = (await db.execute( + select(CohortAuditEvent) + .where(CohortAuditEvent.action == COHORT_ACTION_TOPOLOGY_SNAPSHOT) + .order_by(CohortAuditEvent.created_at.desc()) + .limit(1) + )).scalar_one_or_none() + + return { + "isolation_enabled": settings.cohort_isolation_enabled, + "default_policy": settings.cohort_default_policy, + "preflight_error": preflight_error, + "preview": { + aid: (None if g is None else sorted(g)) for aid, g in gates.items() + }, + "summary": summarise_gates(gates), + "bot_names": {r.agent_id: r.bot_name for r in active}, + "snapshot": snapshot, + } + + +@router.get("/cohorts", response_class=HTMLResponse) +async def admin_cohorts( + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """List all cohorts with member counts, plus the live gate preview.""" + result = await db.execute( + select(Cohort).options(selectinload(Cohort.memberships)).order_by(Cohort.name) + ) + cohorts = result.scalars().unique().all() + + # Creator names for display + creator_map: dict[str, str] = {} + creator_ids = {c.created_by for c in cohorts if c.created_by} + if creator_ids: + u_result = await db.execute(select(User).where(User.id.in_(creator_ids))) + for u in u_result.scalars().all(): + creator_map[str(u.id)] = u.name + + return templates.TemplateResponse( + request, + "admin/cohorts.html", + _template_context( + request, + current_user, + active_admin="cohorts", + cohorts=cohorts, + creator_map=creator_map, + error=request.query_params.get("error"), + notice=request.query_params.get("notice"), + gate=await _cohort_gate_context(db), + ), + ) + + +@router.post("/cohorts/create") +async def admin_cohort_create( + request: Request, + name: str = Form(...), + description: str = Form(""), + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Create a new cohort.""" + name = name.strip().lower() + if not _COHORT_NAME_RE.match(name): + return RedirectResponse( + url="/admin/cohorts?error=Invalid+name+(lowercase+letters,+numbers,+hyphens;+max+48)", + status_code=302, + ) + existing = await db.execute(select(Cohort).where(Cohort.name == name)) + if existing.scalar_one_or_none(): + return RedirectResponse( + url="/admin/cohorts?error=A+cohort+with+that+name+already+exists", + status_code=302, + ) + cohort = Cohort( + name=name, + description=description.strip() or None, + created_by=current_user.id, + ) + db.add(cohort) + await db.flush() + await record_cohort_audit_event( + db, + action=COHORT_ACTION_CREATED, + cohort_id=cohort.id, + cohort_name=cohort.name, + actor=current_user, + ) + await db.commit() + return RedirectResponse(url=f"/admin/cohorts/{cohort.id}", status_code=302) + + +@router.get("/cohorts/topology", response_class=HTMLResponse) +async def admin_cohort_topology( + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Agent x cohort matrix — edit the whole topology in one pass. + + Granular control: every (agent, cohort) pair is a checkbox, so an admin can move + several agents across several cohorts in one save instead of walking the + per-cohort add/remove forms. The resulting per-agent gate is shown alongside, + computed with the engine's own logic. See v2 §12. + + Registered before /cohorts/{cohort_id} so "topology" is not swallowed as a UUID + path parameter. + """ + cohorts = (await db.execute( + select(Cohort).order_by(Cohort.name) + )).scalars().all() + agents = (await db.execute( + select(AgentRegistry).order_by(AgentRegistry.bot_name) + )).scalars().all() + rows = (await db.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + membership_set = {f"{c}:{a}" for c, a in rows} + + return templates.TemplateResponse( + request, + "admin/cohort_topology.html", + _template_context( + request, + current_user, + active_admin="cohorts", + cohorts=cohorts, + agents=agents, + membership_set=membership_set, + error=request.query_params.get("error"), + notice=request.query_params.get("notice"), + gate=await _cohort_gate_context(db), + ), + ) + + +@router.post("/cohorts/topology") +async def admin_cohort_topology_save( + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Apply a whole-matrix edit as a diff against the cells that were rendered. + + The form posts one ``cell`` value per ticked box (``{cohort_id}:{agent_id}``), + one ``present_agent`` per rendered row and one ``present_cohort`` per rendered + column; the rendered cell set is their cross product, which is what the + template renders (an unconditional nested loop). Sending markers instead of one + hidden input per cell keeps the payload at agents+cohorts fields rather than + agents*cohorts — 60x56 posted 3,528 fields and hit Starlette's + ``max_fields=1000``, which is why the matrix could not be saved at all. + + Diffing against ``rendered`` rather than against the whole table means a stale + or partial form can never delete memberships for a cohort or agent it did not + display — the usual checkbox-matrix data-loss bug. Unknown cohort/agent ids are + ignored, never written. Every add and remove is audited individually. + + ``present_cohort``/``present_agent`` are filtered down to ids that still exist + *before* the cross product is built, not after: the product of two + attacker-controlled lists is multiplicative, so crossing them first and + validating each resulting cell afterward (the naive approach) lets a payload + well within ``_TOPOLOGY_MAX_FIELDS`` build a cross product many orders of + magnitude larger than either list — e.g. 25,000 garbage ids on each side is + 50,000 form fields (under the cap) but a 625-million-entry ``rendered`` set. + Filtering first bounds the product by the real ``Cohort``/``AgentRegistry`` row + counts instead. ``ticked`` is filtered the same way for the same reason, and so + that a ticked cell naming an id that no longer exists is silently ignored + (as it always was) rather than tripping the "malformed submission" guard below, + which is reserved for a cell that names two otherwise-valid ids but was never + part of the rendered cross product at all. + """ + form = await request.form(max_fields=_TOPOLOGY_MAX_FIELDS) + ticked = {v for v in form.getlist("cell") if isinstance(v, str)} + present_agents = {v for v in form.getlist("present_agent") if isinstance(v, str)} + present_cohorts = {v for v in form.getlist("present_cohort") if isinstance(v, str)} + # Checked on the raw, unfiltered marker sets: a genuinely empty submission (no + # rows or no columns rendered at all) is an error, but a submission naming only + # since-deleted rows/columns is not — that is just every cell turning out inert, + # handled below by the (empty) diff loop, not by this guard. + if not present_agents or not present_cohorts: + return RedirectResponse( + url="/admin/cohorts/topology?error=Nothing+to+save", status_code=302 + ) + + cohorts_by_id = { + str(c.id): c for c in (await db.execute(select(Cohort))).scalars().all() + } + valid_agents = { + r[0] for r in (await db.execute(select(AgentRegistry.agent_id))).all() + } + + def _known_cell(cell: str) -> bool: + cid, _, aid = cell.partition(":") + return bool(cid) and bool(aid) and cid in cohorts_by_id and aid in valid_agents + + # Filter BEFORE crossing: bounds the cross product by the current table sizes + # rather than by the (attacker-controlled) lengths of the submitted lists. + present_cohorts &= cohorts_by_id.keys() + present_agents &= valid_agents + ticked = {t for t in ticked if _known_cell(t)} + + rendered = {f"{cid}:{aid}" for cid in present_cohorts for aid in present_agents} + if ticked - rendered: + return RedirectResponse( + url="/admin/cohorts/topology?error=Malformed+submission", status_code=302 + ) + + existing = { + (str(cid), aid): mid + for mid, cid, aid in (await db.execute( + select(CohortMembership.id, CohortMembership.cohort_id, + CohortMembership.agent_id) + )).all() + } + + added = removed = 0 + for cell in sorted(rendered): + cid, _, aid = cell.partition(":") + if not cid or not aid or cid not in cohorts_by_id or aid not in valid_agents: + continue # stale form referencing something that no longer exists + want = cell in ticked + have = (cid, aid) in existing + if want and not have: + db.add(CohortMembership( + cohort_id=uuid.UUID(cid), agent_id=aid, added_by=current_user.id, + )) + await record_cohort_audit_event( + db, + action=COHORT_ACTION_AGENT_ADDED, + cohort_id=uuid.UUID(cid), + cohort_name=cohorts_by_id[cid].name, + agent_id=aid, + actor=current_user, + ) + added += 1 + elif have and not want: + await db.execute( + sa_delete(CohortMembership).where( + CohortMembership.id == existing[(cid, aid)] + ) + ) + await record_cohort_audit_event( + db, + action=COHORT_ACTION_AGENT_REMOVED, + cohort_id=uuid.UUID(cid), + cohort_name=cohorts_by_id[cid].name, + agent_id=aid, + actor=current_user, + ) + removed += 1 + + if added or removed: + await db.commit() + return RedirectResponse( + url=f"/admin/cohorts/topology?notice={added}+added,+{removed}+removed", + status_code=302, + ) + + +@router.get("/cohorts/{cohort_id}", response_class=HTMLResponse) +async def admin_cohort_detail( + cohort_id: uuid.UUID, + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Cohort detail: members, add-agent picker, agent->cohort map, audit log.""" + result = await db.execute( + select(Cohort).options(selectinload(Cohort.memberships)).where(Cohort.id == cohort_id) + ) + cohort = result.scalar_one_or_none() + if not cohort: + raise HTTPException(status_code=404, detail="Cohort not found") + + # All agents, for the add-agent picker and status display. + agents_result = await db.execute( + select(AgentRegistry).order_by(AgentRegistry.bot_name) + ) + all_agents = agents_result.scalars().all() + agent_by_id = {a.agent_id: a for a in all_agents} + + member_ids = {m.agent_id for m in cohort.memberships} + available_agents = [a for a in all_agents if a.agent_id not in member_ids] + + # Adder names for the members table. + adder_map: dict[str, str] = {} + adder_ids = {m.added_by for m in cohort.memberships if m.added_by} + if adder_ids: + u_result = await db.execute(select(User).where(User.id.in_(adder_ids))) + for u in u_result.scalars().all(): + adder_map[str(u.id)] = u.name + + # Read-only agent -> cohorts map (all memberships across all cohorts). + all_memberships = (await db.execute( + select(CohortMembership.agent_id, Cohort.name) + .join(Cohort, CohortMembership.cohort_id == Cohort.id) + )).all() + agent_cohort_map: dict[str, list[str]] = {} + for aid, cname in all_memberships: + agent_cohort_map.setdefault(aid, []).append(cname) + + # Audit log for this cohort. Matched on cohort_id, which outlives the cohort + # row; a recreated cohort with the same name gets a new id and so a fresh + # trail, which is the honest reading. + audit_events = (await db.execute( + select(CohortAuditEvent) + .where(CohortAuditEvent.cohort_id == cohort_id) + .order_by(CohortAuditEvent.created_at.desc()) + .limit(200) + )).scalars().all() + + return templates.TemplateResponse( + request, + "admin/cohort_detail.html", + _template_context( + request, + current_user, + active_admin="cohorts", + cohort=cohort, + agent_by_id=agent_by_id, + available_agents=available_agents, + adder_map=adder_map, + all_agents=all_agents, + agent_cohort_map=agent_cohort_map, + audit_events=audit_events, + error=request.query_params.get("error"), + notice=request.query_params.get("notice"), + gate=await _cohort_gate_context(db), + ), + ) + + +@router.post("/cohorts/{cohort_id}/delete") +async def admin_cohort_delete( + cohort_id: uuid.UUID, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Delete a cohort. Refused while it still has members. + + A server-side guard, not just a disabled button: deleting a populated cohort + cascades its memberships away, silently reshaping the interaction topology of a + running simulation. Remove the members first so each removal is an audited, + individually reversible step. See v2 §12. + + A cohort id that does not exist is a 404, matching every other route in this + module whose path-addressed row is missing (and ``admin_cohort_detail`` for + this very id). It used to be a bare redirect to the list, which said nothing + at all — a double-submitted delete looked like it had done the work. + """ + result = await db.execute( + select(Cohort).options(selectinload(Cohort.memberships)).where(Cohort.id == cohort_id) + ) + cohort = result.scalar_one_or_none() + if not cohort: + raise HTTPException(status_code=404, detail="Cohort not found") + if cohort.memberships: + return RedirectResponse( + url=f"/admin/cohorts/{cohort_id}?error=Remove+all+" + f"{len(cohort.memberships)}+members+before+deleting+this+cohort", + status_code=302, + ) + name = cohort.name + await record_cohort_audit_event( + db, + action=COHORT_ACTION_DELETED, + cohort_id=cohort_id, + cohort_name=name, + actor=current_user, + ) + await db.delete(cohort) + await db.commit() + return RedirectResponse( + url=f"/admin/cohorts?notice=Deleted+cohort+{name}", status_code=302 + ) + + +@router.post("/cohorts/{cohort_id}/add-agent") +async def admin_cohort_add_agent( + cohort_id: uuid.UUID, + agent_id: str = Form(...), + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Add an agent to the cohort.""" + result = await db.execute(select(Cohort).where(Cohort.id == cohort_id)) + cohort = result.scalar_one_or_none() + if not cohort: + raise HTTPException(status_code=404, detail="Cohort not found") + + agent_id = agent_id.strip().lower() + # Validate the agent exists in the registry. + agent_exists = await db.execute( + select(AgentRegistry.id).where(AgentRegistry.agent_id == agent_id) + ) + if not agent_exists.scalar_one_or_none(): + return RedirectResponse( + url=f"/admin/cohorts/{cohort_id}?error=Unknown+agent", + status_code=302, + ) + # Reject duplicate membership. + dup = await db.execute( + select(CohortMembership.id).where( + CohortMembership.cohort_id == cohort_id, + CohortMembership.agent_id == agent_id, + ) + ) + if dup.scalar_one_or_none(): + return RedirectResponse( + url=f"/admin/cohorts/{cohort_id}?error=Agent+is+already+a+member", + status_code=302, + ) + db.add(CohortMembership( + cohort_id=cohort_id, + agent_id=agent_id, + added_by=current_user.id, + )) + await record_cohort_audit_event( + db, + action=COHORT_ACTION_AGENT_ADDED, + cohort_id=cohort_id, + cohort_name=cohort.name, + agent_id=agent_id, + actor=current_user, + ) + await db.commit() + return RedirectResponse(url=f"/admin/cohorts/{cohort_id}", status_code=302) + + +@router.post("/cohorts/{cohort_id}/remove-agent") +async def admin_cohort_remove_agent( + cohort_id: uuid.UUID, + agent_id: str = Form(...), + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_admin_user), +): + """Remove an agent from the cohort. + + An unknown cohort id is a 404, as everywhere else in this module: the old + behaviour redirected to ``/admin/cohorts/{cohort_id}``, a detail page that + then 404s itself — so the user paid for two requests to be told nothing. + Removing an agent that is not a member is a different case and stays a quiet + redirect back to the (real) detail page: a stale Remove button is a race the + admin cannot act on, and the page it returns to already shows the truth. + """ + cohort = (await db.execute( + select(Cohort).where(Cohort.id == cohort_id) + )).scalar_one_or_none() + if not cohort: + raise HTTPException(status_code=404, detail="Cohort not found") + result = await db.execute( + select(CohortMembership).where( + CohortMembership.cohort_id == cohort_id, + CohortMembership.agent_id == agent_id.strip().lower(), + ) + ) + membership = result.scalar_one_or_none() + if membership: + await record_cohort_audit_event( + db, + action=COHORT_ACTION_AGENT_REMOVED, + cohort_id=cohort_id, + cohort_name=cohort.name, + agent_id=membership.agent_id, + actor=current_user, + ) + await db.delete(membership) + await db.commit() + return RedirectResponse(url=f"/admin/cohorts/{cohort_id}", status_code=302) diff --git a/src/routers/agent_page.py b/src/routers/agent_page.py index e42e2bf..3e6b238 100644 --- a/src/routers/agent_page.py +++ b/src/routers/agent_page.py @@ -1,14 +1,17 @@ """My Agent page router.""" +import asyncio import logging import re import uuid +from datetime import UTC from pathlib import Path from fastapi import APIRouter, Depends, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates -from sqlalchemy import distinct, func, select +from sqlalchemy import distinct, func, select, tuple_ +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload @@ -24,7 +27,7 @@ ThreadDecision, User, ) -from src.services.profile_export import export_private_profile, export_profile_to_markdown +from src.services.profile_export import export_profile_to_markdown from src.services.validators import is_valid_email logger = logging.getLogger(__name__) @@ -37,6 +40,30 @@ "zt-3sxfrrisw-t4hRz4aMfZZPxThxUaTGKA" ) +# Thread roots per page. The window's unit is threads, not messages: replies no +# longer consume slots, so this surfaces more distinct conversations than the +# previous flat 100-message window did. +_ROOT_LIMIT = 50 + + +async def _visible_channels(db: AsyncSession, run_id, aid: str) -> list[str]: + """Channels this agent participates in (has authored a message in), plus + #general. + + Shared by ``agent_conversations`` and ``agent_thread_replies`` — the + channel set is one of the thread-expand endpoint's four authorization + axes (``channel_name.in_(channels)`` on the root re-resolution query), not + just a display filter, so it must be computed identically in both places + rather than copy-pasted and left free to drift. + """ + ch_rows = await db.execute( + select(distinct(AgentMessage.channel_name)).where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.agent_id == aid, + ) + ) + return sorted({r[0] for r in ch_rows} | {"general"}) + def _extract_proposal_title(text: str | None) -> str: """Best-effort one-line title for a proposal summary. @@ -285,8 +312,13 @@ async def agent_dashboard( delegates = [] if agent.delegate_slack_ids: from src.services.slack_tokens import get_any_bot_token - delegates = _resolve_delegate_names( - agent.delegate_slack_ids, await get_any_bot_token(db) + # to_thread because _resolve_delegate_names is sync and calls + # slack_web.get_user_info once per delegate, each of which can retry with + # backoff. Run inline it would block the event loop for every other + # request the process is serving, not just this dashboard render. + delegates = await asyncio.to_thread( + _resolve_delegate_names, + agent.delegate_slack_ids, await get_any_bot_token(db), ) # Pending invitations (for PI view) @@ -358,6 +390,30 @@ def _user_slack_id_in_list(user: User, slack_ids: list[str]) -> bool: # -------------------------------------------------------------------------- +async def derive_agent_identity( + db: AsyncSession, full_name: str +) -> tuple[str, str]: + """Return ``(agent_id, bot_name)`` for a PI's display name. + + Both values are derived here, together, because they must agree: the + collision prefix used to be applied to agent_id at one line and bot_name + rebuilt from the bare last name four lines later, so Peng Wu got + ``pwu`` / ``WuBot`` — colliding with Chunlei Wu's bot while the ids differed. + CLAUDE.md documents ``pwu`` / ``PWuBot``. + """ + last_name = full_name.split()[-1] + stem = "".join(c for c in last_name.lower() if c.isalpha()) + display = last_name + + collision = await db.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == stem) + ) + if collision.scalar_one_or_none(): + initial = full_name[0] + return f"{initial.lower()}{stem}", f"{initial.upper()}{display}Bot" + return stem, f"{display}Bot" + + @router.post("/request") async def request_agent( request: Request, @@ -374,20 +430,12 @@ async def request_agent( if existing.scalar_one_or_none(): return RedirectResponse(url="/agent", status_code=302) - last_name = current_user.name.split()[-1].lower() - agent_id = "".join(c for c in last_name if c.isalpha()) - - collision = await db.execute( - select(AgentRegistry).where(AgentRegistry.agent_id == agent_id) - ) - if collision.scalar_one_or_none(): - first_initial = current_user.name[0].lower() - agent_id = f"{first_initial}{agent_id}" + agent_id, bot_name = await derive_agent_identity(db, current_user.name) agent = AgentRegistry( agent_id=agent_id, user_id=current_user.id, - bot_name=f"{current_user.name.split()[-1]}Bot", + bot_name=bot_name, pi_name=current_user.name, status="pending", ) @@ -499,9 +547,13 @@ async def reopen_proposal( # Reopening re-injects the agent into a live discussion (posts guidance to # Slack / spins up a private refinement channel), so it is blocked while the - # agent is inactive — exactly the cross-cohort interaction inactivation is + # agent is inactive — exactly the interaction that inactivating an agent is # meant to stop. Reactivate the agent to reopen proposals for further # discussion. (Unlike `review`, this requires status == 'active'.) + # + # Note: the reopen flow creates a collab_private channel, and the cohort gate + # deliberately exempts those — a PI explicitly pairing two agents outranks an + # admin-level cohort grouping. See .notes/cohort-system-v2.md §7. if agent.status != "active": raise HTTPException( status_code=403, @@ -583,41 +635,59 @@ async def reopen_proposal( # Legacy fallback: flag is off → post guidance verbatim to the origin # public thread. This reproduces the pre-refactor behavior and is the # same code as before; kept gated so rollback is a config change. - try: - from slack_sdk import WebClient - - from src.services.slack_tokens import token_for_agent_row - bot_token = token_for_agent_row(agent) - if not bot_token: - raise HTTPException(status_code=500, detail="No bot token available") - client = WebClient(token=bot_token) - channels_result = client.conversations_list( - types="public_channel,private_channel", limit=200, - ) - channel_id = None - for ch in channels_result.get("channels", []): - if ch["name"] == td.channel: - channel_id = ch["id"] - break - if not channel_id: - raise HTTPException(status_code=500, detail=f"Channel #{td.channel} not found") - client.chat_postMessage( - channel=channel_id, - text=f"*PI guidance from {current_user.name}:*\n\n{guidance}", - thread_ts=td.thread_id, - ) - logger.warning( - "LEGACY PATH: PI %s posted guidance in proposal thread %s via %s " - "(enable_private_refinement=False)", - current_user.name, td.thread_id, agent.agent_id, - ) - except HTTPException: - raise - except Exception as exc: - logger.error("Failed to post PI guidance to Slack: %s", exc) - raise HTTPException( - status_code=500, detail=f"Failed to post to Slack: {str(exc)[:100]}", - ) + from src.services.slack_tokens import slack_globally_enabled, token_for_agent_row + + if not await slack_globally_enabled(db): + # Slack off → write the guidance to the DB inbox on the origin thread. + from src.services.pi_inbox import get_latest_run_id, record_pi_message + run_id = await get_latest_run_id(db) + if run_id: + await record_pi_message( + db, run_id=run_id, channel_name=td.channel, + content=f"PI guidance from {current_user.name}: {guidance}", + sender_name=f"{current_user.name} (PI)", thread_ts=td.thread_id, + ) + logger.info("Reopen guidance for %s written to DB inbox (Slack off)", td.thread_id) + else: + try: + # The channel lookup goes through the boundary. It used to read a + # single 200-item page of the paginated conversations.list, so a + # workspace with more channels than that reported "Channel not + # found" for a channel that exists; list_channel_ids follows every + # cursor and raises rather than returning a subset. Archived + # channels are counted deliberately — this asks "which id owns + # this name", not "can the bot join it". + # + # The post goes through it too, threaded: post_message takes + # thread_ts precisely so this caller does not need a raw client. + # It also splits at 4000 characters, which the raw call did not — + # long PI guidance was silently chunked by Slack. + from src.services.slack_web import list_channel_ids_async, post_message_async + + bot_token = token_for_agent_row(agent) + if not bot_token: + raise HTTPException(status_code=500, detail="No bot token available") + channel_id = (await list_channel_ids_async(bot_token)).get(td.channel) + if not channel_id: + raise HTTPException(status_code=500, detail=f"Channel #{td.channel} not found") + await post_message_async( + bot_token, + channel_id, + f"*PI guidance from {current_user.name}:*\n\n{guidance}", + thread_ts=td.thread_id, + ) + logger.warning( + "LEGACY PATH: PI %s posted guidance in proposal thread %s via %s " + "(enable_private_refinement=False)", + current_user.name, td.thread_id, agent.agent_id, + ) + except HTTPException: + raise + except Exception as exc: + logger.error("Failed to post PI guidance to Slack: %s", exc) + raise HTTPException( + status_code=500, detail=f"Failed to post to Slack: {str(exc)[:100]}", + ) existing = await db.execute( select(ProposalReview).where( @@ -653,6 +723,346 @@ async def reopen_proposal( # -------------------------------------------------------------------------- +@router.get("/{agent_id}/conversations", response_class=HTMLResponse) +async def agent_conversations( + agent_id: str, + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Read view of the agent's recent conversations + a form to post a message. + + This is the Slack-independent way for a PI to see what their agent is + discussing and to inject a message/tag — it writes to the DB inbox, which + the running simulation ingests. See specs/local-db-conversations.md. + """ + from src.services.conversation_feed import own_or_gated, resolve_agent_gate + from src.services.pi_inbox import get_latest_run_id + + agent, is_owner = await get_agent_with_access(agent_id, db, current_user) + if agent.status not in ("active", "inactive"): + return RedirectResponse(url="/agent", status_code=302) + aid = agent.agent_id + + run_id = await get_latest_run_id(db) + channels: list[str] = [] + messages: list[dict] = [] + dms: list[dict] = [] + if run_id: + from src.models import PiDmMessage + dm_rows = await db.execute( + select(PiDmMessage) + .where( + PiDmMessage.simulation_run_id == run_id, + PiDmMessage.agent_id == aid, + ) + # Total ordering. posted_at alone is not one: pi_dm_messages.posted_at + # carries server_default '0' (migration 0020), so any writer that omits + # it produces a tie group, and with LIMIT the tie makes row SELECTION + # plan-dependent, not just row order. + .order_by(PiDmMessage.posted_at.desc(), PiDmMessage.created_at.desc(), + PiDmMessage.id.desc()) + .limit(20) + ) + dms = [ + {"direction": d.direction, "sender": d.sender_name or "", "content": d.content} + for d in reversed(dm_rows.scalars().all()) + ] + channels = await _visible_channels(db, run_id, aid) + # What this PI may read == what their bot may act on. Filtering happens in + # SQL, before LIMIT: #general carries every other cohort's traffic, so + # filtering in Python afterwards would leave the page nearly empty. + gate = await resolve_agent_gate(db, aid) + # own_or_gated (src/services/conversation_feed.py) is gate_clause widened + # with the PI's own-post carve-out — see its docstring for why the OR is + # needed. One expression here, in the reply-count query below, and in + # agent_thread_replies keeps the feed, the badge, and the expansion from + # ever disagreeing on what a PI may see. + gated = own_or_gated(gate, aid) + + # Thread ROOTS, newest first. `phase` is belt-and-braces alongside + # `thread_ts IS NULL`; the two agree on every row. + # + # The three-column ordering is load-bearing, not stylistic. Migration + # 0019 adds posted_at with server_default '0', so EVERY row that + # predates it shares one value. With `ORDER BY posted_at DESC LIMIT + # 50` over a tie group larger than 50, Postgres is free to return any + # 50 — measured on a 200-row tie group, the index-scan and seq-scan + # plans returned two DISJOINT pages, so half the messages were + # unreachable and which half flipped with the plan. Adding created_at + # and the primary key makes the sort total, so the page is stable and + # every row is reachable by paging. + root_rows = await db.execute( + select(AgentMessage) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.channel_name.in_(channels), + AgentMessage.thread_ts.is_(None), + AgentMessage.phase == "new_post", + gated, + ) + .order_by(AgentMessage.posted_at.desc(), AgentMessage.created_at.desc(), + AgentMessage.id.desc()) + .limit(_ROOT_LIMIT) + ) + roots = list(reversed(root_rows.scalars().all())) + + # Reply counts, gated with the SAME clause (including the own-post + # carve-out) so the badge can never promise turns the expansion will not + # show. The real invariant a reply query must honour is that a reply + # lives in ITS ROOT's channel — `uq_agent_messages_run_ts` + # (src/models/agent_activity.py) only proves root ids don't collide + # across channels within a run; it says nothing about where a reply + # naming that root as `thread_ts` was posted. Nothing else enforces that + # a `collab_private` reply's `thread_ts` can't coincide with a public + # root's `message_ts` — and `gate_clause`'s unconditional + # `collab_private` pass would let such a reply count toward (and, via + # the expand endpoint, render into) a conversation it does not belong + # to. So this matches `(thread_ts, channel_name)` pairs against each + # root's own channel, not `thread_ts` alone. + root_pairs = [(r.message_ts, r.channel_name) for r in roots if r.message_ts] + counts: dict[str, int] = {} + if root_pairs: + count_rows = await db.execute( + select(AgentMessage.thread_ts, func.count(AgentMessage.id)) + .where( + AgentMessage.simulation_run_id == run_id, + tuple_(AgentMessage.thread_ts, AgentMessage.channel_name).in_(root_pairs), + gated, + ) + .group_by(AgentMessage.thread_ts) + ) + counts = {ts: n for ts, n in count_rows} + + messages = [ + { + "channel": m.channel_name, + "sender": m.sender_name or (m.agent_id or "PI"), + "is_bot": m.is_bot, + "content": m.content, + "message_ts": m.message_ts, + "thread_ts": m.thread_ts, + "reply_count": counts.get(m.message_ts, 0), + "posted_at": m.posted_at, + } + for m in roots + ] + else: + channels = ["general"] + + return templates.TemplateResponse( + request, + "agent/conversations.html", + _template_context( + request, current_user, agent=agent, is_owner=is_owner, + channels=channels, messages=messages, dms=dms, + has_run=run_id is not None, + posted=request.query_params.get("posted"), + ), + ) + + +@router.get("/{agent_id}/thread/{message_ts}", response_class=HTMLResponse) +async def agent_thread_replies( + agent_id: str, + message_ts: str, + request: Request, + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Replies for one thread, as an HTML fragment for the conversations page. + + ``message_ts`` is a guessable identifier, so authorisation cannot stop at the + agent: the ROOT is re-resolved under this agent's channel set and cohort gate + before any reply is read. Anything that does not resolve is a 404 — absent, + not-a-root, another channel, and out-of-cohort are deliberately + indistinguishable to the caller. + + Replies are gated too, with the same clause that produced the count on the + page (``own_or_gated``), so the badge and the expansion can never disagree. + This diverges from the engine, which classifies ``get_thread_history`` as + UNGATED (``src/agent/message_log.py:224-226``) because it is thread-internal; + here the whole point is that out-of-cohort traffic must not become reachable + by clicking, and a future reader should not "fix" this back toward engine + parity. + """ + from src.services.conversation_feed import own_or_gated, resolve_agent_gate + from src.services.pi_inbox import get_latest_run_id + + agent, _is_owner = await get_agent_with_access(agent_id, db, current_user) + if agent.status not in ("active", "inactive"): + raise HTTPException(status_code=404) + aid = agent.agent_id + + run_id = await get_latest_run_id(db) + if not run_id: + raise HTTPException(status_code=404) + + channels = await _visible_channels(db, run_id, aid) + + gate = await resolve_agent_gate(db, aid) + gated = own_or_gated(gate, aid) + root = (await db.execute( + select(AgentMessage) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.message_ts == message_ts, + AgentMessage.thread_ts.is_(None), + AgentMessage.phase == "new_post", + AgentMessage.channel_name.in_(channels), + gated, + ) + .limit(1) + )).scalar_one_or_none() + if root is None: + raise HTTPException(status_code=404) + + # Scoped to the root's OWN channel, not just `thread_ts` — see the count + # query's comment in `agent_conversations` for why `thread_ts` alone is not + # the invariant a reply query can rely on. + reply_rows = await db.execute( + select(AgentMessage) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.thread_ts == message_ts, + AgentMessage.channel_name == root.channel_name, + gated, + ) + .order_by(AgentMessage.posted_at.asc(), AgentMessage.created_at.asc(), + AgentMessage.id.asc()) + ) + replies = [ + { + "sender": m.sender_name or (m.agent_id or "PI"), + "is_bot": m.is_bot, + "content": m.content, + } + for m in reply_rows.scalars().all() + ] + + return templates.TemplateResponse( + request, "agent/_thread_replies.html", {"replies": replies} + ) + + +@router.post("/{agent_id}/message") +async def post_agent_message( + agent_id: str, + request: Request, + channel_name: str = Form(...), + content: str = Form(...), + thread_ts: str = Form(""), + tag_bot: str = Form(""), + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Write a PI-authored message into the DB inbox for the agent's workspace. + + Ingested by the running simulation via _poll_inbound_from_db — the + Slack-independent equivalent of a PI posting in a Slack channel. + """ + from src.services.pi_inbox import ( + get_latest_run_id, + pi_may_post_to_channel, + record_pi_message, + ) + + agent, is_owner = await get_agent_with_access(agent_id, db, current_user) + if agent.status != "active": + raise HTTPException(status_code=403, detail="Agent is not active") + + text = content.strip() + if not text: + raise HTTPException(status_code=400, detail="Message cannot be empty") + # Optionally address the PI's own bot so it engages (same @BotName convention + # the Slack path uses; the engine's tag detection is identical). + if tag_bot and f"@{agent.bot_name.lower()}" not in text.lower(): + text = f"@{agent.bot_name} {text}" + + run_id = await get_latest_run_id(db) + if not run_id: + raise HTTPException(status_code=409, detail="No simulation run to post into yet") + + # `channel_name` is form input, so it can name any channel in the run — + # including another pair's collab_private refinement channel. The DB-only + # path has no Slack ACL to fall back on, so authorization is checked here + # against private_channel_members. See specs/privacy-and-channel-visibility.md. + target_channel = channel_name.strip() or "general" + if not await pi_may_post_to_channel( + db, + run_id=run_id, + channel_name=target_channel, + user_id=current_user.id, + agent_id=agent.agent_id, + ): + raise HTTPException(status_code=403, detail="Not a member of that channel") + + async def _write() -> None: + await record_pi_message( + db, + run_id=run_id, + channel_name=target_channel, + content=text, + sender_name=f"{current_user.name} (PI)", + thread_ts=thread_ts.strip() or None, + ) + await db.commit() + + # M1b guard: the canonical id can collide with another process (the sim) + # minting the same microsecond for this run, which hits the + # uq_agent_messages_run_ts constraint and would otherwise surface as a raw + # 500. Roll back and retry once — record_pi_message mints a fresh, monotonic + # id, so the retry gets a new ts. See PR #19 review M1. + try: + await _write() + except IntegrityError: + await db.rollback() + try: + await _write() + except IntegrityError: + await db.rollback() + raise HTTPException( + status_code=409, + detail="Message could not be saved due to a conflict, please retry", + ) + logger.info("[%s] PI %s posted a web message to #%s", agent_id, current_user.name, channel_name) + return RedirectResponse(url=f"/agent/{agent_id}/conversations?posted=1", status_code=302) + + +@router.post("/{agent_id}/dm") +async def send_agent_dm( + agent_id: str, + request: Request, + content: str = Form(...), + db: AsyncSession = Depends(get_db), + current_user: User = Depends(get_current_user), +): + """Send a DM directive to the agent's bot (standing instruction / question). + + Writes an inbound pi_dm_messages row; the sim processes it via + _poll_pi_dms_from_db (same path as a Slack DM). See specs/local-db-conversations.md. + """ + from src.services.pi_inbox import get_latest_run_id, record_pi_dm, web_pi_user_id + + agent, is_owner = await get_agent_with_access(agent_id, db, current_user) + if agent.status != "active": + raise HTTPException(status_code=403, detail="Agent is not active") + text = content.strip() + if not text: + raise HTTPException(status_code=400, detail="Message cannot be empty") + run_id = await get_latest_run_id(db) + if not run_id: + raise HTTPException(status_code=409, detail="No simulation run yet") + await record_pi_dm( + db, run_id=run_id, agent_id=agent_id, + pi_user_id=web_pi_user_id(current_user.id), direction="inbound", + content=text, sender_name=f"{current_user.name} (PI)", + ) + await db.commit() + logger.info("[%s] PI %s sent a web DM directive", agent_id, current_user.name) + return RedirectResponse(url=f"/agent/{agent_id}/conversations?posted=1", status_code=302) + + @router.get("/{agent_id}/profile", response_class=HTMLResponse) async def view_private_profile( agent_id: str, @@ -911,23 +1321,25 @@ async def connect_slack( error = None try: - from slack_sdk import WebClient from src.services.slack_tokens import get_any_bot_token + from src.services.slack_web import lookup_user_by_email_async bot_token = await get_any_bot_token(db) if not bot_token: error = "No Slack bot token available to perform lookup." else: - client = WebClient(token=bot_token) - result = client.users_lookupByEmail(email=email) - slack_user_id = result["user"]["id"] + # The boundary translates Slack's users_not_found into None, so "no + # such user" is a return value here rather than a substring match on + # an exception message. + slack_user_id = await lookup_user_by_email_async(bot_token, email) + if not slack_user_id: + error = ( + f"No Slack user found with email {email}. " + "Have you joined the workspace first?" + ) except Exception as exc: - error_msg = str(exc) - if "users_not_found" in error_msg: - error = f"No Slack user found with email {email}. Have you joined the workspace first?" - else: - logger.warning("Slack lookup failed for %s: %s", email, exc) - error = f"Slack lookup failed: {error_msg[:100]}" + logger.warning("Slack lookup failed for %s: %s", email, exc) + error = f"Slack lookup failed: {str(exc)[:100]}" if slack_user_id: agent.slack_user_id = slack_user_id @@ -941,20 +1353,30 @@ async def connect_slack( def _resolve_delegate_names(slack_ids: list[str], bot_token: str | None) -> list[dict]: - """Resolve Slack user IDs to display names using the given bot token.""" - from slack_sdk import WebClient + """Resolve Slack user IDs to display names using the given bot token. + + A name that will not resolve falls back to the raw id — this only feeds the + dashboard's delegate list, so one unresolvable id must not blank the rest. + """ + from src.services.slack_web import get_user_info + if not bot_token: return [{"slack_id": sid, "name": sid} for sid in slack_ids] - client = WebClient(token=bot_token) delegates = [] for sid in slack_ids: + info = None try: - info = client.users_info(user=sid) - name = info["user"].get("real_name") or info["user"].get("name") or sid - delegates.append({"slack_id": sid, "name": name}) - except Exception: - delegates.append({"slack_id": sid, "name": sid}) + # Returns None for a user Slack does not know, so the fallback below + # covers both "no such user" and a failed call. + info = get_user_info(bot_token, sid) + except Exception as exc: + logger.warning("Could not resolve Slack display name for %s: %s", sid, exc) + info = info or {} + delegates.append({ + "slack_id": sid, + "name": info.get("real_name") or info.get("name") or sid, + }) return delegates @@ -981,27 +1403,36 @@ async def delegate_connect_slack( error = None try: - from slack_sdk import WebClient from src.services.slack_tokens import get_any_bot_token + from src.services.slack_web import lookup_user_by_email_async + bot_token = await get_any_bot_token(db) if not bot_token: error = "No Slack bot token available." else: - client = WebClient(token=bot_token) - result = client.users_lookupByEmail(email=current_user.email) - sid = result["user"]["id"] - current_ids = list(agent.delegate_slack_ids or []) - if sid not in current_ids: - current_ids.append(sid) - agent.delegate_slack_ids = current_ids - await db.commit() - return RedirectResponse(url=f"/agent/{agent_id}/dashboard", status_code=302) + # None means Slack has no such user (the boundary translates + # users_not_found), so the "join the workspace first" message is + # driven by a value rather than by a substring of an exception. + sid = await lookup_user_by_email_async(bot_token, current_user.email) + if not sid: + error = ( + f"No Slack account found for {current_user.email}. " + "Please join the workspace first." + ) + else: + current_ids = list(agent.delegate_slack_ids or []) + if sid not in current_ids: + current_ids.append(sid) + agent.delegate_slack_ids = current_ids + await db.commit() + return RedirectResponse( + url=f"/agent/{agent_id}/dashboard", status_code=302 + ) except Exception as exc: - error_msg = str(exc) - if "users_not_found" in error_msg: - error = f"No Slack account found for {current_user.email}. Please join the workspace first." - else: - error = f"Slack lookup failed: {error_msg[:100]}" + logger.warning( + "Delegate Slack lookup failed for %s: %s", current_user.email, exc + ) + error = f"Slack lookup failed: {str(exc)[:100]}" return RedirectResponse( url=f"/agent/{agent_id}/dashboard?slack_error=" + (error or "Unknown error"), @@ -1025,7 +1456,7 @@ async def invite_delegate( """Send delegate invitation(s) by email.""" import re import secrets - from datetime import datetime, timedelta, timezone + from datetime import datetime, timedelta from src.config import get_settings from src.models import DelegateInvitation @@ -1092,7 +1523,7 @@ async def invite_delegate( email=email, token=token, status="pending", - expires_at=datetime.now(timezone.utc) + timedelta(days=30), + expires_at=datetime.now(UTC) + timedelta(days=30), ) db.add(invitation) await db.flush() # Get the ID @@ -1169,19 +1600,18 @@ async def remove_delegate( # Remove Slack ID if present if delegate.user.email and agent.delegate_slack_ids: try: - from slack_sdk import WebClient from src.services.slack_tokens import get_any_bot_token + from src.services.slack_web import lookup_user_by_email_async + bot_token = await get_any_bot_token(db) if bot_token: - client = WebClient(token=bot_token) - slack_result = client.users_lookupByEmail(email=delegate.user.email) - sid = slack_result["user"]["id"] + sid = await lookup_user_by_email_async(bot_token, delegate.user.email) current_ids = list(agent.delegate_slack_ids or []) - if sid in current_ids: + if sid and sid in current_ids: current_ids.remove(sid) agent.delegate_slack_ids = current_ids if current_ids else None - except Exception: - pass # Slack sync is best-effort + except Exception as exc: + logger.warning("Delegate Slack sync is best-effort; skipped: %s", exc) await db.delete(delegate) await db.commit() diff --git a/src/routers/invite.py b/src/routers/invite.py index de295dc..00949b6 100644 --- a/src/routers/invite.py +++ b/src/routers/invite.py @@ -1,7 +1,7 @@ """Invitation acceptance router.""" import logging -from datetime import datetime, timezone +from datetime import UTC, datetime from fastapi import APIRouter, Depends, Request from fastapi.responses import HTMLResponse, RedirectResponse @@ -56,7 +56,7 @@ async def accept_invite( ) # Check expiry - if invitation.expires_at < datetime.now(timezone.utc): + if invitation.expires_at < datetime.now(UTC): if invitation.status == "pending": invitation.status = "expired" await db.commit() @@ -145,7 +145,7 @@ async def confirm_accept_invite( {"request": request, "error": "This invitation is no longer valid."}, ) - if invitation.expires_at < datetime.now(timezone.utc): + if invitation.expires_at < datetime.now(UTC): invitation.status = "expired" await db.commit() return templates.TemplateResponse( @@ -198,7 +198,7 @@ async def _accept_invitation( # Already a delegate — just mark invitation and redirect invitation.status = "accepted" invitation.accepted_by_user_id = user.id - invitation.accepted_at = datetime.now(timezone.utc) + invitation.accepted_at = datetime.now(UTC) await db.commit() # Get agent_id for redirect @@ -221,7 +221,7 @@ async def _accept_invitation( # Mark invitation accepted invitation.status = "accepted" invitation.accepted_by_user_id = user.id - invitation.accepted_at = datetime.now(timezone.utc) + invitation.accepted_at = datetime.now(UTC) # Try Slack sync agent_result = await db.execute( @@ -231,19 +231,25 @@ async def _accept_invitation( if user.email: try: - from slack_sdk import WebClient - from src.routers.agent_page import _get_bot_token - bot_token = _get_bot_token() + from src.services.slack_tokens import token_for_agent_row + from src.services.slack_web import lookup_user_by_email_async + + bot_token = token_for_agent_row(agent) if bot_token: - client = WebClient(token=bot_token) - slack_result = client.users_lookupByEmail(email=user.email) - sid = slack_result["user"]["id"] - current_ids = list(agent.delegate_slack_ids or []) - if sid not in current_ids: - current_ids.append(sid) - agent.delegate_slack_ids = current_ids - except Exception: - pass # Slack sync is best-effort + sid = await lookup_user_by_email_async(bot_token, user.email) + if sid: + current_ids = list(agent.delegate_slack_ids or []) + if sid not in current_ids: + current_ids.append(sid) + agent.delegate_slack_ids = current_ids + except Exception as exc: + # Best-effort by design (specs/web-delegates.md §Slack Linkage): a + # delegate is useful without a Slack id. But LOG it — a bare `pass` + # here hid an ImportError for an unknown length of time, and the + # whole sync was dead code with nothing to show for it. + logger.warning( + "Delegate Slack-ID sync failed for agent %s: %s", agent.agent_id, exc + ) await db.commit() diff --git a/src/routers/onboarding.py b/src/routers/onboarding.py index 2833109..065bd7a 100644 --- a/src/routers/onboarding.py +++ b/src/routers/onboarding.py @@ -10,8 +10,8 @@ from src.database import get_db from src.dependencies import get_current_user -from src.routers.auth import pop_post_login_redirect from src.models import AgentRegistry, Job, ResearcherProfile, User +from src.routers.auth import pop_post_login_redirect from src.services.profile_export import ( PRIVATE_PROFILES_DIR, export_private_profile, @@ -163,8 +163,8 @@ def parse_list(val: str) -> list[str]: agent_id_for_export = agent_reg.agent_id if agent_reg else None # Export to markdown for agent consumption (include publications) - from src.services.profile_export import export_profile_to_markdown from src.models import Publication + from src.services.profile_export import export_profile_to_markdown pub_result = await db.execute( select(Publication).where(Publication.user_id == current_user.id) ) @@ -314,43 +314,6 @@ async def save_private_profile( return RedirectResponse(url="/profile?onboarding_complete=1", status_code=302) -@router.post("/complete") -async def complete_onboarding( - request: Request, - db: AsyncSession = Depends(get_db), - current_user: User = Depends(get_current_user), -): - """Mark onboarding as complete.""" - was_complete = current_user.onboarding_complete - current_user.onboarding_complete = True - await db.commit() - - _maybe_send_welcome(current_user, was_complete) - - pending_token = request.session.pop("pending_invite_token", None) - if pending_token: - request.session.pop("post_login_redirect", None) - return RedirectResponse(url=f"/invite/{pending_token}", status_code=302) - - # Resume the page the user originally requested before being sent to login. - next_url = pop_post_login_redirect(request) - if next_url: - return RedirectResponse(url=next_url, status_code=302) - return RedirectResponse(url="/profile?onboarding_complete=1", status_code=302) - - -@router.get("/done", response_class=HTMLResponse) -async def onboarding_done( - request: Request, - current_user: User = Depends(get_current_user), -): - return templates.TemplateResponse( - request, - "onboarding/complete.html", - _template_context(request, current_user), - ) - - @router.post("/retry") async def retry_pipeline( request: Request, diff --git a/src/routers/profile.py b/src/routers/profile.py index 79a89b5..f50ddf3 100644 --- a/src/routers/profile.py +++ b/src/routers/profile.py @@ -68,7 +68,6 @@ async def profile_view( current_user, profile=profile, publications=publications, - pending_profile=profile.pending_profile if profile else None, just_completed_onboarding=onboarding_complete, ), ) diff --git a/src/routers/public.py b/src/routers/public.py index 14d1bcf..948c67b 100644 --- a/src/routers/public.py +++ b/src/routers/public.py @@ -92,7 +92,7 @@ def _render_graph(request: Request, context: dict) -> HTMLResponse: # hardcoded grouping of agent_ids by institution (the agent roster itself now # lives in the AgentRegistry table). _SCRIPPS = { - # Active Cabo cohort + # Active Cabo run window "su", "wiseman", "grotjahn", "ward", "briney", "forli", "lairson", "badran", "kern", "lasker", "lippi", "maillie", "millar", "miller", "mravic", "paulson", "pwu", "seiple", "williamson", "wilson", @@ -112,13 +112,13 @@ def _render_graph(request: Request, context: dict) -> HTMLResponse: "nomura": "UC Berkeley", } -# Cohort cutover for the Cabo retreat graph: matches commit 0ef4741 +# Run-window cutover for the Cabo retreat graph: matches commit 0ef4741 # (the Cabo retreat roster reshape). All proposals to date share a single -# simulation_run_id, so date is the only way to isolate the new cohort. -CABO_COHORT_START = datetime(2026, 3, 1, tzinfo=timezone.utc) +# simulation_run_id, so date is the only way to isolate the new run window. +CABO_WINDOW_START = datetime(2026, 3, 1, tzinfo=timezone.utc) # Schultz alumni pilot = the PIs seeded from newuserlist01.tsv + newuserlist02.tsv -# (formerly "cohort 001"). These are matched by ORCID (the identifier the lists +# (formerly "cohort 001" — the run-window naming below supersedes it). These are matched by ORCID (the identifier the lists # are keyed on) rather than agent_id, since agent_id collision-prefixing # (cliu/liu, schen/chen, ckim/kim, wliu/wu, achatterjee/chatterjee) makes # hand-deriving IDs fragile. The last three entries of newuserlist02.tsv had a @@ -164,18 +164,18 @@ def _render_graph(request: Request, context: dict) -> HTMLResponse: "SPARSE-1527BF6A", # Sida Shao (null ORCID in TSV; resolved from DB) }) -# Three time-scoped cohorts of the same long-running simulation. Edges are +# Three time-scoped run windows of the same long-running simulation. Edges are # bounded by proposal *decided_at* (window_end exclusive). The post-creation -# join boundary (cohort_start) sits EARLIER than the decision window, because a +# join boundary (window_start_bound) sits EARLIER than the decision window, because a # thread can be opened a couple days before its proposal lands (e.g. the group # proposals decided Jun 6 came from posts created Jun 4). Bounding posts to the # decision window would silently drop those edges. See memory -# project_graph_cohort_windows. +# the window constants below. # -# Cabo cohort: Apr 27 – May 7, 2026 (inlined in the /cabo-graph route) +# Cabo window: Apr 27 – May 7, 2026 (inlined in the /cabo-graph route) # Schultz alumni pilot: Jun 1 – Jun 4, 2026 # Schultz group alumni: Jun 5 – Jun 10, 2026 -JUNE_POST_START = datetime(2026, 6, 1, tzinfo=timezone.utc) # post boundary for both June cohorts +JUNE_POST_START = datetime(2026, 6, 1, tzinfo=timezone.utc) # post boundary for both June windows SCHULTZ_PILOT_START = datetime(2026, 6, 1, tzinfo=timezone.utc) SCHULTZ_PILOT_END = datetime(2026, 6, 5, tzinfo=timezone.utc) # exclusive: through Jun 4 SCHULTZ_GROUP_START = datetime(2026, 6, 5, tzinfo=timezone.utc) @@ -191,7 +191,7 @@ def _institution_for(agent_id: str) -> str: # --------------------------------------------------------------------------- -# Institution canonicalization for the cohort views. +# Institution canonicalization for the run-window views. # # Profiles store free-text institutions ("Scripps Research", "The Scripps # Research Institute", "UCSF Medical Center", ...). We group them so the same @@ -576,7 +576,7 @@ async def _build_graph_payload( scripps_only: bool = False, all_agents: bool = False, orcids: frozenset[str] | None = None, - cohort_start: datetime = CABO_COHORT_START, + window_start_bound: datetime = CABO_WINDOW_START, window_start: datetime | None = None, window_end: datetime | None = None, use_profile_institution: bool = False, @@ -596,12 +596,12 @@ async def _build_graph_payload( actually appear in an in-window edge. - otherwise: active agents only. - ``cohort_start`` bounds which *posts* count (their ``created_at``), keeping - edges inside the right cohort/run. + ``window_start_bound`` bounds which *posts* count (their ``created_at``), keeping + edges inside the right run window. ``window_start`` / ``window_end`` bound when a proposal was *decided*, letting a caller scope to an arbitrary date range (e.g. a single retreat - week). ``window_start`` defaults to ``cohort_start``; ``window_end`` is + week). ``window_start`` defaults to ``window_start_bound``; ``window_end`` is exclusive and unbounded when ``None``. The window is applied to ``decided_at`` only — not to post creation — so a proposal decided in the window still counts even if its thread was opened earlier. @@ -612,7 +612,7 @@ async def _build_graph_payload( ``largest_component_only`` (default True) trims the result to the single largest connected component — sensible for a dense graph, but it hides - isolated proposal dyads, so pass False early in a cohort when proposals are + isolated proposal dyads, so pass False early in a window when proposals are still disconnected pairs. """ if orcids is not None: @@ -656,8 +656,8 @@ async def _build_graph_payload( else: institution_of = lambda row: _institution_for(row.agent_id) # noqa: E731 - decided_floor = window_start or cohort_start - params = {"cohort_start": cohort_start, "decided_floor": decided_floor} + decided_floor = window_start or window_start_bound + params = {"window_start_bound": window_start_bound, "decided_floor": decided_floor} window_end_clause = "" if window_end is not None: window_end_clause = " AND decided_at < :window_end" @@ -666,11 +666,11 @@ async def _build_graph_payload( edges_result = await db.execute( text( f""" - WITH cohort_posts AS ( + WITH window_posts AS ( SELECT message_ts FROM agent_messages WHERE phase = 'new_post' - AND created_at >= :cohort_start + AND created_at >= :window_start_bound AND message_ts IS NOT NULL ), pairs AS ( @@ -685,7 +685,7 @@ async def _build_graph_payload( WHERE outcome = 'proposal' AND origin_visibility = 'public' AND decided_at >= :decided_floor{window_end_clause} - AND thread_id IN (SELECT message_ts FROM cohort_posts) + AND thread_id IN (SELECT message_ts FROM window_posts) ), -- The agent-only proposal for a thread is the FIRST one the bots -- reached. Any later row on the same thread is a re-proposal after a @@ -880,7 +880,7 @@ async def schultz_alumni_pilot(request: Request, db: AsyncSession = Depends(get_ nodes, links = await _cached_graph_payload( db, orcids=SCHULTZ_PILOT_ORCIDS, - cohort_start=JUNE_POST_START, + window_start_bound=JUNE_POST_START, window_start=SCHULTZ_PILOT_START, window_end=SCHULTZ_PILOT_END, use_profile_institution=True, @@ -917,7 +917,7 @@ async def schultz_group_alumni(request: Request, db: AsyncSession = Depends(get_ nodes, links = await _cached_graph_payload( db, all_agents=True, - cohort_start=JUNE_POST_START, + window_start_bound=JUNE_POST_START, window_start=SCHULTZ_GROUP_START, window_end=SCHULTZ_GROUP_END, use_profile_institution=True, @@ -1075,9 +1075,12 @@ async def update_proposal_vote_details( if vote_obj is None: raise HTTPException(status_code=404, detail="unknown vote") - # Light ownership check: if the row has a token, a provided one must match. + # Light ownership check: if the row has a token, the caller must supply the + # matching one. A caller that simply omits `voter_token` gets `token is + # None`, which must NOT satisfy the check — omitting it is not a way to + # bypass ownership on a row that has a token. token = _clean_token(payload.voter_token) - if vote_obj.voter_token and token and vote_obj.voter_token != token: + if vote_obj.voter_token and vote_obj.voter_token != token: raise HTTPException(status_code=403, detail="token mismatch") vote_obj.details = _clean_details(payload.details) diff --git a/src/services/cohorts.py b/src/services/cohorts.py new file mode 100644 index 0000000..bc2b1a7 --- /dev/null +++ b/src/services/cohorts.py @@ -0,0 +1,218 @@ +"""Cohort gate semantics — one implementation, two callers. + +The simulation engine applies the gate; the admin UI previews it. They must never +disagree, so the decision logic lives here as pure functions over plain data rather +than inside ``SimulationEngine``. That is also what makes the semantics testable +without an engine, a database, or a running loop. + +See .notes/cohort-system-v2.md §5 (gate semantics) and §12 (admin preview). +""" + +from __future__ import annotations + +import logging +import uuid +from collections.abc import Hashable, Iterable, Mapping, Sequence +from typing import Any + +logger = logging.getLogger(__name__) + +# Policy values for settings.cohort_default_policy. +POLICY_OPEN = "open" +POLICY_ISOLATED = "isolated" + + +def preflight_reason( + *, + isolation_enabled: bool, + policy: str, + cohort_count: int, + has_db: bool, + live_members: int | None = None, +) -> str | None: + """Return why isolation must be forced OFF, or None when it may run. + + Enabling isolation must never silently silence the roster: + + 1. Without a database handle memberships cannot be read, so the flag would + appear to work while doing nothing. + 2. Under ``policy="isolated"``, if no agent on the live roster has any cohort + membership, every agent is uncohorted and therefore isolated — roster-wide + silence. That covers both the obvious case (zero cohorts, the state a fresh + deployment is in) and the easy mistake of creating a cohort and never adding + anyone to it. Counting *live members* rather than cohorts is the check that + actually corresponds to the hazard. + + ``live_members`` is the number of roster agents with at least one membership. + It defaults to None for callers that only have the cohort count, in which case + the cohort count is used as the weaker proxy. + + See v2 §5.3. + """ + if not isolation_enabled: + return None + if not has_db: + return ( + "no database session available — cohort memberships cannot be read, " + "so isolation would silently do nothing" + ) + if policy == POLICY_ISOLATED: + effective = cohort_count if live_members is None else live_members + if effective == 0: + detail = ( + "zero cohorts defined" if cohort_count == 0 + else f"{cohort_count} cohort(s) defined but no live agent is a member" + ) + return ( + f"cohort_default_policy='isolated' with {detail} would isolate every " + "agent (roster-wide silence). Add agents to a cohort, or use " + "cohort_default_policy='open'" + ) + return None + + +def compute_gates( + *, + membership_rows: Iterable[tuple[Hashable, str]], + agent_ids: Sequence[str], + isolation_enabled: bool, + policy: str, + cohort_count: int, + has_db: bool = True, +) -> tuple[dict[str, set[str] | None], str | None]: + """Compute each agent's ``allowed_sender_ids`` plus any preflight refusal. + + ``membership_rows`` is ``(cohort_id, agent_id)`` pairs — the whole + ``cohort_memberships`` table. ``agent_ids`` is the *live roster*, but it only + gates which agents receive an entry of their OWN in the returned ``gates`` + dict — an agent absent from it simply has no running process asking for its + gate. It does **not** otherwise filter membership rows: ``members_by_cohort`` + below is built from the raw rows, unfiltered by roster membership, so a + membership naming an agent that is not currently running still lands in + every cohort-mate's ``mates`` set and is treated as an allowed sender by + everyone else in its cohort(s) — it does NOT simply vanish. Confirmed live: a + membership-only agent_id with 56 memberships and no ``AgentRegistry`` row + passes every one of its cohort-mates' gates. If a non-roster membership + should be inert, that has to be enforced by removing the row (or filtering + ``membership_rows`` before calling this), not assumed from this function. + + Returns ``(gates, preflight_error)`` where a gate value is: + + - ``None`` — no filtering for this agent (gate off); + - a set — the bot senders this agent may act on. Empty only under + ``policy="isolated"`` for an uncohorted agent. + + Truth table (v2 §5.1 / §5.2): + + ============================ =================== ======================= + isolation / policy agent has cohorts? gate + ============================ =================== ======================= + disabled — None + enabled, preflight refused — None + enabled yes union of co-members + enabled, policy "open" no None + enabled, policy "isolated" no set() + ============================ =================== ======================= + """ + # Materialise first: the preflight needs to know how many LIVE agents actually + # have a membership, not just how many cohorts exist. + members_by_cohort: dict[Hashable, set[str]] = {} + cohorts_by_agent: dict[str, set[Hashable]] = {} + for cohort_id, agent_id in membership_rows: + members_by_cohort.setdefault(cohort_id, set()).add(agent_id) + cohorts_by_agent.setdefault(agent_id, set()).add(cohort_id) + live_members = sum(1 for aid in agent_ids if cohorts_by_agent.get(aid)) + + reason = preflight_reason( + isolation_enabled=isolation_enabled, + policy=policy, + cohort_count=cohort_count, + has_db=has_db, + live_members=live_members, + ) + if not isolation_enabled or reason is not None: + return {aid: None for aid in agent_ids}, reason + + isolate_uncohorted = policy == POLICY_ISOLATED + + # Under policy "open", an uncohorted agent is unrestricted — and that has to hold + # in BOTH directions. Its own gate is None, so it may act on anyone; but a cohorted + # agent's gate is the union of its co-members, which would never contain it. The + # result was an agent that could react and never be replied to: it could not hold a + # conversation, which is the opposite of "unrestricted". Adding the uncohorted + # agents to every cohorted agent's mate set implements the §5.1 row + # ("`A` has no cohort memberships, and policy = open -> Yes") and makes the + # relation symmetric. + # + # Found by a real multi-turn run: an uncohorted agent opened two threads and no + # cohorted agent ever replied. The gate-computation tests all passed, and the + # symmetry test skipped the case (it compared only pairs where BOTH gates were + # sets). See v2 §5.2. + unrestricted: set[str] = ( + set() if isolate_uncohorted + else {aid for aid in agent_ids if not cohorts_by_agent.get(aid)} + ) + + gates: dict[str, set[str] | None] = {} + for aid in agent_ids: + cohort_ids = cohorts_by_agent.get(aid) + if not cohort_ids: + # policy "open": unrestricted. Never an empty set here — that was the + # inverted v1 behaviour that silenced uncohorted agents (v2 §5.4). + gates[aid] = set() if isolate_uncohorted else None + continue + mates: set[str] = set() + for cid in cohort_ids: + mates |= members_by_cohort.get(cid, set()) + gates[aid] = mates | unrestricted + return gates, None + + +def summarise_gates(gates: Mapping[str, set[str] | None]) -> dict[str, Any]: + """Counts for logging and the admin banner.""" + gated = [aid for aid, g in gates.items() if g is not None] + isolated = sorted(aid for aid, g in gates.items() if g is not None and not g) + return { + "total": len(gates), + "gated": len(gated), + "isolated": isolated, + "unrestricted": sorted(aid for aid, g in gates.items() if g is None), + } + + +async def record_cohort_audit_event( + db: Any, + *, + action: str, + cohort_name: str, + cohort_id: uuid.UUID | None = None, + agent_id: str | None = None, + actor: Any | None = None, + simulation_run_id: uuid.UUID | None = None, + topology: dict[str, Any] | None = None, + commit: bool = False, +) -> None: + """Append one row to ``cohort_audit_events``. + + Denormalises ``cohort_name`` and the actor's email so the trail survives the + cohort being deleted and the user row going away. Never raises: an audit write + failing must not take down the mutation it describes — but it is logged loudly, + because a silently missing trail is worse than a noisy one. + """ + try: + from src.models import CohortAuditEvent + + db.add(CohortAuditEvent( + cohort_id=cohort_id, + cohort_name=cohort_name, + agent_id=agent_id, + action=action, + actor_id=getattr(actor, "id", None), + actor_email=getattr(actor, "email", None), + simulation_run_id=simulation_run_id, + topology=topology, + )) + if commit: + await db.commit() + except Exception as exc: # pragma: no cover - defensive + logger.error("[cohort] audit write failed (%s on %s): %s", action, cohort_name, exc) diff --git a/src/services/conversation_feed.py b/src/services/conversation_feed.py new file mode 100644 index 0000000..59594b5 --- /dev/null +++ b/src/services/conversation_feed.py @@ -0,0 +1,149 @@ +"""What a PI may see in their agent's conversations feed. + +The simulation engine gates what each agent may *act on* (``_entry_allowed`` in +``src/agent/message_log.py``); this module gates what that agent's PI may *read* +on the web page. They are the same rule, and they must never disagree — the same +constraint ``src/services/cohorts.py`` was written under, and for the same reason. + +``_entry_allowed`` filters ``LogEntry`` objects already in memory. The page cannot +do that: the filter has to run in SQL, before ``LIMIT``, or ``#general`` traffic +from every other cohort consumes the window and the page comes back near-empty. +So the rule is expressed twice — once as a predicate, once as a WHERE fragment — +and ``tests/integration/test_conversation_feed.py`` asserts the two agree on +every row of the engine's own decision table. + +Three functions, one pipeline: ``resolve_agent_gate`` computes *what the gate +is* for the viewing agent, by calling the engine's own ``compute_gates`` +(``src/services/cohorts.py``) — the same call ``_cohort_gate_context`` in +``src/routers/admin.py`` makes for the admin preview, so the page can never +compute a different gate than the engine would. ``gate_clause`` then turns that +gate into the SQL predicate above, and ``own_or_gated`` widens it with a PI's +own-post carve-out (see its docstring). The one deliberate difference from the +admin preview: ``resolve_agent_gate``'s roster is the active agents **plus the +viewing agent**, because ``/agent/{id}/conversations`` also admits an inactive +viewer, and ``compute_gates`` only returns a gate for agents in the roster it is +handed. +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import ColumnElement, and_, false, func, or_, select, true +from sqlalchemy.ext.asyncio import AsyncSession + +from src.config import get_settings +from src.models import AgentMessage, AgentRegistry, Cohort, CohortMembership +from src.services.cohorts import compute_gates +from src.visibility import VISIBILITY_COLLAB_PRIVATE + +logger = logging.getLogger(__name__) + + +def gate_clause(gate: set[str] | None) -> ColumnElement[bool]: + """The cohort gate as a SQL predicate over ``AgentMessage``. + + Mirrors ``_entry_allowed`` clause for clause, in the same order, so the two + can be diffed by eye: + + - ``gate is None`` — no filtering for this agent (isolation off, or policy + "open" and the agent is uncohorted); + - the author is a **human** — keyed on ``is_bot``, *not* on a NULL + ``agent_id``. ``agent_messages.agent_id`` is nullable, so a bot-authored row + with a NULL ``agent_id`` would otherwise pass through the human bypass; + - the row is in a ``collab_private`` channel — a PI explicitly paired those + agents, and an admin-level grouping must not veto an explicit human pairing; + - a bot row with a NULL ``agent_id`` cannot be attributed to a cohort, so it + fails closed; + - otherwise the author must share a cohort with the viewing agent. + + ``gate`` is an EMPTY set for an uncohorted agent under + ``cohort_default_policy="isolated"``. That is the one input where the + membership branch must be dropped entirely rather than rendered as an empty + ``IN`` — hence the ``if gate else false()``. + """ + if gate is None: + return true() + return or_( + AgentMessage.is_bot.is_(False), + AgentMessage.visibility == VISIBILITY_COLLAB_PRIVATE, + and_( + AgentMessage.agent_id.is_not(None), + AgentMessage.agent_id.in_(gate), + ) if gate else false(), + ) + + +def own_or_gated(gate: set[str] | None, agent_id: str) -> ColumnElement[bool]: + """``gate_clause`` widened with a PI's own-post carve-out. + + A PI must always see their OWN bot's posts, even when that bot is active + but not yet placed in a cohort — under ``policy="isolated"`` that agent's + gate is the empty set (see ``resolve_agent_gate``/``compute_gates``), and + ``gate_clause(set())`` admits nothing from the membership branch, so + without this OR the PI's own posts would vanish the moment their bot is + activated and before an admin has assigned it a cohort. This is a + deliberate, safe divergence from ``_entry_allowed``: the engine never + needs this clause because an agent is never asked to decide whether to + act on its own post. Safe because it can only ever admit THIS agent's own + rows, never another agent's. + + One expression serves three call sites that must never drift apart: the + conversations feed's roots query, its reply-count query, and the thread + expand endpoint's root re-resolution and reply fetch. + """ + return or_(gate_clause(gate), AgentMessage.agent_id == agent_id) + + +async def resolve_agent_gate(db: AsyncSession, agent_id: str) -> set[str] | None: + """The viewing agent's ``allowed_sender_ids``, via the engine's own computation. + + Same call the admin preview makes (``_cohort_gate_context``), with one + deliberate difference: the roster is the active agents **plus the viewing + agent**. ``/agent/{id}/conversations`` admits ``status in ("active", + "inactive")``, but ``compute_gates`` only returns keys for the roster it is + handed. Widening the roster here is what stops an inactive viewer falling + through ``gates.get(agent_id)`` below to its default of ``None`` — i.e. + silently getting an UNGATED feed — rather than an ergonomic nicety to dodge + a ``KeyError`` (``dict.get`` never raises). Adding the viewer can only + *raise* ``live_members``, which the preflight compares against zero — so it + cannot turn a refusal into a silent roster-wide isolation. + """ + settings = get_settings() + roster = { + r[0] for r in (await db.execute( + select(AgentRegistry.agent_id).where(AgentRegistry.status == "active") + )).all() + } + roster.add(agent_id) + rows = (await db.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + cohort_count = (await db.execute( + select(func.count()).select_from(Cohort) + )).scalar() or 0 + + gates, preflight_error = compute_gates( + membership_rows=[(r[0], r[1]) for r in rows], + agent_ids=sorted(roster), + isolation_enabled=settings.cohort_isolation_enabled, + policy=settings.cohort_default_policy, + cohort_count=cohort_count, + has_db=True, + ) + if preflight_error is not None and settings.cohort_isolation_enabled: + # This is the fail-open engine semantics `compute_gates`/`preflight_reason` + # deliberately implement (src/services/cohorts.py) — do NOT change it here. + # But it is silent by default: an admin can reach this state with one click + # ("all / none" on every column of /admin/cohorts/topology, then save), and + # nothing short of this line says so. Every gate this call resolves is + # `None` while the condition holds, which means every PI conversations feed + # and thread-expand read is UNGATED — the isolation the operator turned on + # is not applying to any agent, not just this one. + logger.warning( + "[conversation_feed] preflight forced the cohort gate OFF for agent " + "%r (%s) — PI-facing reads via resolve_agent_gate are consequently " + "UNGATED for every agent until this is resolved", + agent_id, preflight_error, + ) + return gates.get(agent_id) diff --git a/src/services/email.py b/src/services/email.py index f446120..871fc8b 100644 --- a/src/services/email.py +++ b/src/services/email.py @@ -198,6 +198,43 @@ def build_welcome_email(to_email: str, name: str | None = None, user_id: str | N greeting_name = (name or "").strip().split(" ")[0] if name else "" greeting = f"Hi {greeting_name}," if greeting_name else "Hi there," + + # Only describe the reply-by-email review flow when the inbound pipeline + # is actually enabled; otherwise point at the web dashboard alone. + reply_enabled = settings.enable_inbound_email + if reply_enabled: + review_how_text = ( + "HOW PROPOSAL REVIEW WORKS\n" + "When your agent and another lab's agent develop a promising idea, we email\n" + "you a short proposal. You can:\n" + " - Reply with a rating from 1 to 4:\n" + " 1 = Not a good idea 2 = Good idea\n" + " 3 = Great idea 4 = Excellent idea\n" + ' - Reply with instructions (e.g. "focus on the mitochondrial angle") and\n' + " your agent will re-engage to refine the idea.\n" + " - Or review it on the web dashboard.\n" + "Note: while you have unreviewed proposals, your agent pauses new\n" + "conversations — reviewing promptly keeps it active." + ) + review_how_html = ( + "
  • Rate it by replying with a number from 1 to 4.
    • \n" + "
  • Give instructions to refine it, and your agent re-engages.
    • \n" + "
  • Review it on the web dashboard.
    • " + ) + else: + review_how_text = ( + "HOW PROPOSAL REVIEW WORKS\n" + "When your agent and another lab's agent develop a promising idea, we email\n" + "you a short proposal. Open your dashboard to rate it from 1 to 4\n" + "(1 = Not a good idea, 2 = Good idea, 3 = Great idea, 4 = Excellent idea)\n" + "or to give your agent instructions to refine the idea.\n" + "Note: while you have unreviewed proposals, your agent pauses new\n" + "conversations — reviewing promptly keeps it active." + ) + review_how_html = ( + "
  • Rate it from 1 to 4 on your dashboard.
    • \n" + "
  • Give instructions to refine it, and your agent re-engages.
    • " + ) # HTML-escaped greeting for the HTML body (the name is the ORCID display # name, i.e. user-controlled) (SEC-13). greeting_html = f"Hi {esc(greeting_name)}," if greeting_name else "Hi there," @@ -226,17 +263,7 @@ def build_welcome_email(to_email: str, name: str | None = None, user_id: str | N - My Agent ({agent_url}) — request your agent and manage it. - Settings ({settings_url}) — choose which emails you receive and how often. -HOW PROPOSAL REVIEW WORKS -When your agent and another lab's agent develop a promising idea, we email -you a short proposal. You can: - - Reply with a rating from 1 to 4: - 1 = Not a good idea 2 = Good idea - 3 = Great idea 4 = Excellent idea - - Reply with instructions (e.g. "focus on the mitochondrial angle") and - your agent will re-engage to refine the idea. - - Or review it on the web dashboard. -Note: while you have unreviewed proposals, your agent pauses new -conversations — reviewing promptly keeps it active. +{review_how_text} Welcome aboard, The CoPI team — Scripps Research @@ -321,9 +348,7 @@ def build_welcome_email(to_email: str, name: str | None = None, user_id: str | N you a short proposal. You can:

      -
    • Rate it by replying with a number from 1 to 4.
      • -
    • Give instructions to refine it, and your agent re-engages.
      • -
    • Review it on the web dashboard.
      • + {review_how_html}

    1 = Not a good idea • 2 = Good idea • 3 = Great idea • 4 = Excellent idea diff --git a/src/services/email_inbound.py b/src/services/email_inbound.py index 9e86ef2..ba9b189 100644 --- a/src/services/email_inbound.py +++ b/src/services/email_inbound.py @@ -5,7 +5,6 @@ import logging import re import secrets -from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -26,6 +25,39 @@ # Rate limit: max replies per token per hour MAX_REPLIES_PER_TOKEN_PER_HOUR = 10 +# Processing attempts per S3 object before it is quarantined under failed/. +MAX_S3_PROCESS_ATTEMPTS = 3 + +# token -> recent reply timestamps (monotonic-ish epoch seconds). +_RECENT_REPLY_TIMES: dict[str, list[float]] = {} + +# s3 key -> consecutive processing failures (in-memory; resets on restart). +_S3_FAILURE_COUNTS: dict[str, int] = {} + + +def _reply_rate_ok(token: str, now: float | None = None) -> bool: + """Sliding one-hour window per reply token, capped at + MAX_REPLIES_PER_TOKEN_PER_HOUR. In-memory: the worker is a single + long-lived process, and a restart merely resets the window.""" + import time + + ts = time.time() if now is None else now + window = [t for t in _RECENT_REPLY_TIMES.get(token, []) if ts - t < 3600] + if len(window) >= MAX_REPLIES_PER_TOKEN_PER_HOUR: + _RECENT_REPLY_TIMES[token] = window + return False + window.append(ts) + _RECENT_REPLY_TIMES[token] = window + return True + + +def _is_auto_submitted(msg: email.message.Message) -> bool: + """RFC 3834: any Auto-Submitted value other than "no" marks auto-generated + mail (out-of-office replies, list expansions). Processing those — and + answering them with a help email — is how mail loops start.""" + auto = (msg.get("Auto-Submitted") or "").strip().lower() + return bool(auto) and auto != "no" and not auto.startswith("no ") + # Auth verdicts (from the SES-stamped Authentication-Results header) that mean # the message failed a check — any of these on spf/dkim/dmarc rejects the reply. # ("none" is intentionally excluded: it means the sender domain publishes no @@ -51,13 +83,26 @@ def _authentication_results_ok(msg: email.message.Message) -> bool: logger.warning("Rejecting inbound reply: no Authentication-Results header") return False + # Trust ONLY the topmost header. SES prepends its own Authentication- + # Results on receipt, so a sender-forged header always sits below it — + # merging verdicts across all headers ("a pass wins") let a self-stamped + # spf=pass override SES's spf=fail. The topmost header must also carry + # SES's authserv-id: anything else did not transit our SES receipt path. + header = headers[0] + authserv_id = header.split(";", 1)[0].strip().lower() + if authserv_id != "amazonses.com": + logger.warning( + "Rejecting inbound reply: topmost Authentication-Results is from %r, " + "not amazonses.com", + authserv_id, + ) + return False + verdicts: dict[str, str] = {} - for header in headers: - for mech, result in _AUTH_VERDICT_RE.findall(header): - mech_l, result_l = mech.lower(), result.lower() - # Keep the strongest verdict seen for each mechanism (a pass wins). - if mech_l not in verdicts or result_l == "pass": - verdicts[mech_l] = result_l + for mech, result in _AUTH_VERDICT_RE.findall(header): + # First occurrence wins: the leading verdict is the mechanism's result; + # later matches can come from propagated or commented values. + verdicts.setdefault(mech.lower(), result.lower()) for mech in ("spf", "dkim", "dmarc"): if verdicts.get(mech) in _AUTH_FAIL_VERDICTS: @@ -109,10 +154,33 @@ async def poll_inbound_emails(session_factory: async_sessionmaker) -> int: # Delete processed email from S3 s3.delete_object(Bucket=bucket, Key=key) + _S3_FAILURE_COUNTS.pop(key, None) processed += 1 except Exception as exc: logger.error("Error processing inbound email %s: %s", key, exc, exc_info=True) + # A poison message would otherwise be retried every poll + # forever. After MAX_S3_PROCESS_ATTEMPTS consecutive failures, + # quarantine it under failed/ (outside the polled prefix) for + # manual inspection. The counter is in-memory, so a restart + # grants a fresh round of attempts — acceptable. + _S3_FAILURE_COUNTS[key] = _S3_FAILURE_COUNTS.get(key, 0) + 1 + if _S3_FAILURE_COUNTS[key] >= MAX_S3_PROCESS_ATTEMPTS: + try: + failed_key = "failed/" + key.removeprefix(prefix) + s3.copy_object( + Bucket=bucket, + CopySource={"Bucket": bucket, "Key": key}, + Key=failed_key, + ) + s3.delete_object(Bucket=bucket, Key=key) + _S3_FAILURE_COUNTS.pop(key, None) + logger.error( + "Quarantined inbound email %s to %s after %d failed attempts", + key, failed_key, MAX_S3_PROCESS_ATTEMPTS, + ) + except Exception: + logger.error("Failed to quarantine %s", key, exc_info=True) except Exception as exc: logger.error("Error polling inbound emails: %s", exc, exc_info=True) @@ -131,6 +199,12 @@ async def process_inbound_email(raw_email: bytes, db: AsyncSession) -> None: if not _authentication_results_ok(msg): return + # Auto-generated mail (OOO replies, etc.) must never be answered — our + # help email replying to an auto-responder is a mail loop. + if _is_auto_submitted(msg): + logger.info("Ignoring auto-submitted inbound mail (Auto-Submitted header)") + return + # Extract reply token from To header to_addr = msg.get("To", "") token = _extract_reply_token(to_addr) @@ -138,6 +212,12 @@ async def process_inbound_email(raw_email: bytes, db: AsyncSession) -> None: logger.warning("No reply token found in To address: %s", to_addr) return + if not _reply_rate_ok(token): + logger.warning( + "Rate limit exceeded for reply token %s... — dropping reply", token[:8] + ) + return + # Look up notification by token result = await db.execute( select(EmailNotification).where(EmailNotification.reply_token == token) @@ -260,19 +340,50 @@ def _extract_email_address(from_header: str) -> str | None: return None +def _decode_part(part: email.message.Message) -> str: + charset = part.get_content_charset() or "utf-8" + payload = part.get_payload(decode=True) or b"" + return payload.decode(charset, errors="replace") + + +def _html_to_text(html_body: str) -> str: + """Best-effort text extraction for HTML-only replies. + + Quoted history is dropped structurally (

    /gmail_quote) because + the '>' line-prefix convention below only exists in plain text.""" + import html as html_mod + + text = re.sub(r"(?is)<(script|style)\b.*?", "", html_body) + text = re.sub(r'(?is)]*class="[^"]*gmail_quote[^"]*".*', "", text) + text = re.sub(r"(?is)", "", text) + text = re.sub(r"(?i)|

    |", "\n", text) + text = re.sub(r"(?s)<[^>]+>", "", text) + return html_mod.unescape(text) + + def _extract_reply_body(msg: email.message.Message) -> str: - """Extract the reply body, stripping quoted content and signatures.""" + """Extract the reply body, stripping quoted content and signatures. + + Prefers text/plain; falls back to tag-stripped text/html so an HTML-only + reply (some corporate clients) is not silently dropped.""" body = "" + html_body = "" if msg.is_multipart(): for part in msg.walk(): - if part.get_content_type() == "text/plain": - charset = part.get_content_charset() or "utf-8" - body = part.get_payload(decode=True).decode(charset, errors="replace") + ctype = part.get_content_type() + if ctype == "text/plain": + body = _decode_part(part) break + if ctype == "text/html" and not html_body: + html_body = _decode_part(part) + elif msg.get_content_type() == "text/html": + html_body = _decode_part(msg) else: - charset = msg.get_content_charset() or "utf-8" - body = msg.get_payload(decode=True).decode(charset, errors="replace") + body = _decode_part(msg) + + if not body.strip() and html_body: + body = _html_to_text(html_body) # Strip quoted content (lines starting with >) lines = body.split("\n") @@ -504,30 +615,50 @@ async def _handle_instruction( else: # Legacy fallback: flag off → post guidance verbatim to the origin # public thread (same behavior as the web legacy path). - from slack_sdk import WebClient + from src.services.slack_tokens import slack_globally_enabled, token_for_agent_row + + # Slack off → write the guidance to the DB inbox on the origin thread + # instead of posting to Slack. + if not await slack_globally_enabled(db): + from src.services.pi_inbox import get_latest_run_id, record_pi_message + run_id = await get_latest_run_id(db) + if run_id: + await record_pi_message( + db, run_id=run_id, channel_name=td.channel, + content=f"PI guidance from {user.name} (via email): {instruction}", + sender_name=f"{user.name} (PI)", thread_ts=td.thread_id, + ) + logger.info("Email guidance for %s written to DB inbox (Slack off)", td.thread_id) + return True + logger.error("No simulation run to record email guidance for %s", td.thread_id) + return False + + # The channel lookup goes through the boundary. It used to read a + # single 200-item page of the paginated conversations.list, so a + # workspace with more channels than that reported "Channel not found" + # for a channel that exists; list_channel_ids follows every cursor and + # raises rather than returning a subset. + # + # The post goes through it too, threaded: post_message takes thread_ts + # precisely so this caller does not need a raw client. It also splits + # at 4000 characters, which the raw call did not — a long emailed + # instruction was silently chunked by Slack. + from src.services.slack_web import list_channel_ids_async, post_message_async - from src.services.slack_tokens import token_for_agent_row bot_token = token_for_agent_row(agent) if not bot_token: logger.error("No bot token for agent %s", agent.agent_id) return False - client = WebClient(token=bot_token) - channels_result = client.conversations_list( - types="public_channel,private_channel", limit=200 - ) - channel_id = None - for ch in channels_result.get("channels", []): - if ch["name"] == td.channel: - channel_id = ch["id"] - break + channel_id = (await list_channel_ids_async(bot_token)).get(td.channel) if not channel_id: logger.error("Channel #%s not found for instruction posting", td.channel) return False - client.chat_postMessage( - channel=channel_id, - text=f"*PI guidance from {user.name} (via email):*\n\n{instruction}", + await post_message_async( + bot_token, + channel_id, + f"*PI guidance from {user.name} (via email):*\n\n{instruction}", thread_ts=td.thread_id, ) logger.warning( diff --git a/src/services/email_notifications.py b/src/services/email_notifications.py index 9e01e82..bdf26c6 100644 --- a/src/services/email_notifications.py +++ b/src/services/email_notifications.py @@ -331,8 +331,13 @@ async def send_proposal_notification( db.add(notification) await db.flush() - # Build email - reply_to = f"review+{reply_token}@{settings.ses_reply_domain}" + # Build email. Soliciting a reply is only honest when the inbound pipeline + # is actually on — otherwise PIs answer a dead reply domain and get + # silence (this is exactly what happened on prod through 2026-08). + reply_enabled = settings.enable_inbound_email + reply_to = ( + f"review+{reply_token}@{settings.ses_reply_domain}" if reply_enabled else None + ) dashboard_url = f"{settings.base_url}/agent/{agent.agent_id}/dashboard" unsubscribe_token = _generate_unsubscribe_token(str(user.id)) unsubscribe_url = f"{settings.base_url}/settings/unsubscribe/{unsubscribe_token}" @@ -373,25 +378,58 @@ async def send_proposal_notification( f"Review all proposals.

    " ) + if reply_enabled: + review_options_text = ( + f"To review this proposal, you can:\n\n" + f"1. Reply to this email with a rating (1-4) and any comments:\n" + f" 1 = Not a good idea (not interesting, or multiple major weaknesses)\n" + f" 2 = Good idea (medium interest, or one major weakness)\n" + f" 3 = Great idea (high interest, minor weaknesses only)\n" + f" 4 = Excellent idea (high interest, no notable weaknesses)\n\n" + f"2. Reply with instructions for your agent (e.g., \"focus on the\n" + f' mitochondrial angle instead") and it will re-engage to refine\n' + f" the proposal.\n\n" + f"3. Review on the web: {dashboard_url}\n" + ) + else: + review_options_text = ( + f"To review this proposal, rate it on your dashboard: {dashboard_url}\n" + f" 1 = Not a good idea (not interesting, or multiple major weaknesses)\n" + f" 2 = Good idea (medium interest, or one major weakness)\n" + f" 3 = Great idea (high interest, minor weaknesses only)\n" + f" 4 = Excellent idea (high interest, no notable weaknesses)\n" + ) + text_body = ( f"{agent.bot_name} and {other_bot_name} developed a collaboration proposal in #{channel}:\n\n" f"---\n{summary}\n---\n\n" - f"To review this proposal, you can:\n\n" - f"1. Reply to this email with a rating (1-4) and any comments:\n" - f" 1 = Not a good idea (not interesting, or multiple major weaknesses)\n" - f" 2 = Good idea (medium interest, or one major weakness)\n" - f" 3 = Great idea (high interest, minor weaknesses only)\n" - f" 4 = Excellent idea (high interest, no notable weaknesses)\n\n" - f"2. Reply with instructions for your agent (e.g., \"focus on the\n" - f' mitochondrial angle instead") and it will re-engage to refine\n' - f" the proposal.\n\n" - f"3. Review on the web: {dashboard_url}\n" + f"{review_options_text}" f"{backlog_text}\n" f"---\n" f"Unsubscribe: {unsubscribe_url}\n" f"Manage preferences: {settings_url}\n" ) + rating_legend_html = ( + '

    \n' + " 1 = Not a good idea • 2 = Good idea • 3 = Great idea • 4 = Excellent idea\n" + "

    " + ) + if reply_enabled: + review_options_html = ( + '

    Reply to this email to review:

    \n' + '
      \n' + "
    • Rate it with a number 1-4 and any comments
      • \n" + "
    • Give instructions to refine the proposal
      • \n" + "
    \n" + f" {rating_legend_html}" + ) + else: + review_options_html = ( + '

    Rate it on your dashboard:

    \n' + f" {rating_legend_html}" + ) + html_body = email_shell_open() + f"""

    New collaboration proposal

    @@ -402,14 +440,7 @@ async def send_proposal_notification(

    {summary_html}

    -

    Reply to this email to review:

    -
      -
    • Rate it with a number 1-4 and any comments
      • -
    • Give instructions to refine the proposal
      • -
    -

    - 1 = Not a good idea • 2 = Good idea • 3 = Great idea • 4 = Excellent idea -

    + {review_options_html}
    " raw_msg["List-Unsubscribe-Post"] = "List-Unsubscribe=One-Click" @@ -967,7 +999,12 @@ async def _send_new_proposal_email( summary = td.summary_text or "(No summary available)" channel = td.channel or "unknown" - reply_to = f"review+{reply_token}@{settings.ses_reply_domain}" + # Same gating as send_proposal_notification: never solicit a reply while + # the inbound pipeline is off. + reply_enabled = settings.enable_inbound_email + reply_to = ( + f"review+{reply_token}@{settings.ses_reply_domain}" if reply_enabled else None + ) dashboard_url = f"{settings.base_url}/agent/{agent.agent_id}/dashboard" unsubscribe_token = _generate_unsubscribe_token(str(user.id)) unsubscribe_url = f"{settings.base_url}/settings/unsubscribe/{unsubscribe_token}" @@ -982,11 +1019,28 @@ async def _send_new_proposal_email( subject = f"{clean_subject(agent.bot_name)} proposed a collaboration with {clean_subject(other_bot_name)}" + if reply_enabled: + review_line = ( + f"Reply to this email to rate it (1-4) or give your agent instructions, " + f"or review on the web: {dashboard_url}" + ) + review_html = ( + '

    \n' + " Reply to this email to rate it (1–4) or give instructions.\n" + "

    " + ) + else: + review_line = f"Rate it (1-4) on the web: {dashboard_url}" + review_html = ( + '

    \n' + " Rate it (1–4) or give instructions on your dashboard.\n" + "

    " + ) + text_body = ( f"{agent.bot_name} just proposed a collaboration with {other_bot_name} in #{channel}:\n\n" f"---\n{summary}\n---\n\n" - f"Reply to this email to rate it (1-4) or give your agent instructions, " - f"or review on the web: {dashboard_url}\n\n" + f"{review_line}\n\n" f"---\n" f"Unsubscribe: {unsubscribe_url}\n" f"Manage preferences: {settings_url}\n" @@ -999,9 +1053,7 @@ async def _send_new_proposal_email(

    {summary_html}

    -

    - Reply to this email to rate it (1–4) or give instructions. -

    + {review_html}     diff --git a/src/services/grants.py b/src/services/grants.py index 268f699..81bbf3b 100644 --- a/src/services/grants.py +++ b/src/services/grants.py @@ -72,6 +72,14 @@ async def search_opportunities( Returns a list of opportunity dicts with keys: id, number, title, agency, open_date, close_date, description + + ``description`` is **always** ``""``: search2's ``oppHits`` do not carry one. + Measured live 2026-08-04 — a hit's entire key set is agency, agencyCode, + cfdaList, closeDate, docType, id, number, openDate, oppStatus, title. The + real description lives on the detail endpoint (``fetch_opportunity_detail``). + That empty description reaching the drafting prompt is a known, reported bug, + pinned by + ``test_grantbot_live.py::test_the_draft_prompt_is_built_from_an_empty_description``. """ payload = { "keyword": keyword, @@ -99,6 +107,13 @@ async def search_opportunities( "agency": hit.get("agencyCode", ""), "open_date": hit.get("openDate", ""), "close_date": hit.get("closeDate", ""), + # Always "" in practice — search2 sends no description (see the + # docstring). The key is kept deliberately: the drafting prompt in + # grantbot.py reads it, and that empty-description bug is REPORTED + # AND PINNED by + # test_grantbot_live.py::test_the_draft_prompt_is_built_from_an_empty_description. + # Dropping the key here changes "" to a missing key and fixes nothing, + # while breaking the pin that keeps the issue visible. "description": hit.get("description", ""), }) diff --git a/src/services/llm.py b/src/services/llm.py index c3a7f96..7cb2a36 100644 --- a/src/services/llm.py +++ b/src/services/llm.py @@ -164,8 +164,18 @@ async def generate_agent_response( model: str | None = None, max_tokens: int = 1000, log_meta: dict[str, str] | None = None, + on_retry: Callable[[], None] | None = None, ) -> str: - """Generate an agent response via Claude.""" + """Generate an agent response via Claude. + + ``on_retry``, if given, fires once — synchronously, before this returns — + exactly when the max_tokens retry below actually makes a second API call. + A caller that books one call against a rate limiter or budget for this + whole turn (e.g. ``Agent.record_api_call``) should pass that callable here + so a retried turn is booked as the two real API calls it made, not one. + Optional and additive: omitting it changes nothing about behavior or the + return contract. + """ settings = get_settings() model = model or settings.llm_agent_model client = get_anthropic_client() @@ -174,6 +184,7 @@ async def generate_agent_response( message = client.messages.create( model=model, max_tokens=max_tokens, + thinking={"type": "disabled"}, system=system_prompt, messages=messages, ) @@ -208,15 +219,43 @@ async def generate_agent_response( retry_msg = client.messages.create( model=model, max_tokens=retry_max, + thinking={"type": "disabled"}, system=system_prompt, messages=messages, ) + # This is a second real, billed API call for what the caller + # booked as one turn — fire the caller's own accounting hook (if + # any) so a rate limiter sized to "one call per turn" isn't + # quietly undercounting the one turn most likely to retry: the + # phase-5 assessment, whose body runs long enough to hit + # max_tokens before its sidecar at the end. + if on_retry is not None: + on_retry() retry_latency = (time.monotonic() - t0) * 1000 latency_ms += retry_latency if retry_msg.content: response_text = retry_msg.content[0].text message = retry_msg # use retry stats for logging + if message.stop_reason == "max_tokens": + # The retry doubled max_tokens and STILL truncated. The + # retry's (still-truncated) text is returned below — it is + # still the best available answer — but this must be loud: + # for phase 5 the verdict sidecar is + # emitted last, so a still-truncated response silently drops + # the machine-readable verdict while the Slack post can still + # look complete. + agent_id = (log_meta or {}).get("agent_id", "?") + phase = (log_meta or {}).get("phase", "?") + logger.error( + "Response still truncated after 2x max_tokens retry " + "(model=%s agent=%s phase=%s retry_max_tokens=%d " + "out_tok=%d) — returning the truncated text; anything " + "the model emits last (e.g. a phase-5 " + "sidecar) may be missing from it.", + model, agent_id, phase, retry_max, message.usage.output_tokens, + ) + if _call_log_callback and log_meta: from datetime import datetime, timezone _call_log_callback({ @@ -267,6 +306,7 @@ async def generate_with_tools( max_tokens: int = 1000, max_tool_rounds: int = 5, log_meta: dict[str, str] | None = None, + on_retry: Callable[[], None] | None = None, ) -> str: """ Generate a response with Anthropic tool-use API. @@ -275,6 +315,15 @@ async def generate_with_tools( re-call until we get a final text response or hit max_tool_rounds. Returns the final text response. + + ``on_retry``, same contract as ``generate_agent_response``'s: it fires + once — synchronously, before this returns — exactly when one of this + function's two internal max_tokens retries (the "final text" branch's, + or the max-tool-rounds fallback's; at most one runs per call) actually + makes a second API call. A caller that books one call against a rate + limiter for this whole turn (e.g. ``Agent.record_api_call``) should pass + that callable here so a retried turn is booked as the two real API calls + it made, not one. Optional and additive: omitting it changes nothing. """ settings = get_settings() model = model or settings.llm_agent_model @@ -290,6 +339,7 @@ async def generate_with_tools( message = client.messages.create( model=model, max_tokens=max_tokens, + thinking={"type": "disabled"}, system=system_prompt, messages=conversation, tools=tools, @@ -318,9 +368,14 @@ async def generate_with_tools( retry_msg = client.messages.create( model=model, max_tokens=retry_max, + thinking={"type": "disabled"}, system=system_prompt, messages=conversation, ) + # Second real, billed API call for what the caller booked as + # one turn — fire the caller's own accounting hook (if any). + if on_retry is not None: + on_retry() retry_latency = (time.monotonic() - t0) * 1000 latency_ms += retry_latency total_input_tokens += retry_msg.usage.input_tokens @@ -329,8 +384,15 @@ async def generate_with_tools( if retry_texts: response_text = retry_texts[0].text if retry_msg.stop_reason == "max_tokens": - logger.warning( - "Response still truncated after retry (%d tokens)", + agent_id = (log_meta or {}).get("agent_id", "?") + phase = (log_meta or {}).get("phase", "?") + logger.error( + "Response still truncated after 2x max_tokens retry " + "(model=%s agent=%s phase=%s retry_max_tokens=%d " + "out_tok=%d) — returning the truncated text; anything " + "the model emits last (e.g. a closing tag) may be " + "missing from it.", + model, agent_id, phase, retry_max, retry_msg.usage.output_tokens, ) @@ -380,6 +442,7 @@ async def generate_with_tools( message = client.messages.create( model=model, max_tokens=max_tokens, + thinking={"type": "disabled"}, system=system_prompt, messages=conversation, ) @@ -400,9 +463,14 @@ async def generate_with_tools( retry_msg = client.messages.create( model=model, max_tokens=retry_max, + thinking={"type": "disabled"}, system=system_prompt, messages=conversation, ) + # Second real, billed API call for what the caller booked as + # one turn — fire the caller's own accounting hook (if any). + if on_retry is not None: + on_retry() retry_latency = (time.monotonic() - t0) * 1000 latency_ms += retry_latency total_input_tokens += retry_msg.usage.input_tokens @@ -411,6 +479,21 @@ async def generate_with_tools( if retry_texts: response_text = retry_texts[0].text + if retry_msg.stop_reason == "max_tokens": + # This retry site never re-checked stop_reason before this fix: a + # still-truncated response after exhausting max_tool_rounds AND + # doubling max_tokens passed silently. + agent_id = (log_meta or {}).get("agent_id", "?") + phase = (log_meta or {}).get("phase", "?") + logger.error( + "Response still truncated after 2x max_tokens retry " + "(model=%s agent=%s phase=%s retry_max_tokens=%d " + "out_tok=%d) — returning the truncated text; anything " + "the model emits last (e.g. a closing tag) may be " + "missing from it.", + model, agent_id, phase, retry_max, retry_msg.usage.output_tokens, + ) + if _call_log_callback and log_meta: from datetime import datetime, timezone _call_log_callback({ diff --git a/src/services/pi_inbox.py b/src/services/pi_inbox.py new file mode 100644 index 0000000..3be262e --- /dev/null +++ b/src/services/pi_inbox.py @@ -0,0 +1,169 @@ +"""Write PI-authored messages into the DB inbox (Slack-independent input path). + +The agent simulation ingests these rows via SimulationEngine._poll_inbound_from_db, +so a PI can drive their agent with Slack fully off. This is the DB-native +equivalent of the Slack channel-message path. See specs/local-db-conversations.md. +""" + +from __future__ import annotations + +import uuid + +from sqlalchemy import desc, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from src.agent.ids import mint_local_ts +from src.models import ( + VISIBILITY_COLLAB_PRIVATE, + AgentChannel, + AgentMessage, + PiDmMessage, + PrivateChannelMember, + SimulationRun, +) + + +async def get_latest_run_id(db: AsyncSession) -> uuid.UUID | None: + """Return the most recent SimulationRun id, or None if there are no runs.""" + return (await db.execute( + select(SimulationRun.id).order_by(desc(SimulationRun.started_at)).limit(1) + )).scalar_one_or_none() + + +async def _resolve_channel(db: AsyncSession, run_id: uuid.UUID, channel_name: str) -> tuple[str, str]: + """Return (channel_id, visibility) for a channel name in a run. + + Falls back to a local: id / public visibility when the channel has no + agent_channels row yet (e.g. a seeded channel not persisted on an old run). + """ + row = (await db.execute( + select(AgentChannel.channel_id, AgentChannel.visibility) + .where( + AgentChannel.simulation_run_id == run_id, + AgentChannel.channel_name == channel_name, + ) + .limit(1) + )).first() + if row: + return row[0], row[1] + return f"local:{channel_name}", "public" + + +async def pi_may_post_to_channel( + db: AsyncSession, + *, + run_id: uuid.UUID, + channel_name: str, + user_id: uuid.UUID, + agent_id: str, +) -> bool: + """Whether this PI may write into this channel. + + Public channels are open to any PI in the run. ``collab_private`` channels are + not: membership is held in ``private_channel_members`` and is the only thing + standing between a PI and another pair's conversation on the DB-only path — + specs/privacy-and-channel-visibility.md delegates this to Slack ACLs, which + do not exist here. A PI qualifies either in their own right (``user_id``) or + through their bot (``agent_id``); ``removed_at`` is honoured so revoking + membership revokes write access. + + Unknown channel names resolve to public (``_resolve_channel``'s documented + fallback), so they are allowed and land in a ``local:`` channel — the same + behaviour as before this check existed. + """ + row = (await db.execute( + select(AgentChannel.id, AgentChannel.visibility) + .where( + AgentChannel.simulation_run_id == run_id, + AgentChannel.channel_name == channel_name, + ) + .limit(1) + )).first() + if not row: + return True + channel_pk, visibility = row + if visibility != VISIBILITY_COLLAB_PRIVATE: + return True + + member = (await db.execute( + select(PrivateChannelMember.id) + .where( + PrivateChannelMember.agent_channel_id == channel_pk, + PrivateChannelMember.removed_at.is_(None), + or_( + PrivateChannelMember.user_id == user_id, + PrivateChannelMember.agent_id == agent_id, + ), + ) + .limit(1) + )).first() + return member is not None + + +async def record_pi_message( + db: AsyncSession, + *, + run_id: uuid.UUID, + channel_name: str, + content: str, + sender_name: str, + thread_ts: str | None = None, +) -> AgentMessage: + """Insert a human/PI message (is_bot=False) into agent_messages. + + The engine's inbound poller picks it up on its next tick, appends it to the + live MessageLog, and routes it through PI handling (proposal-review clear, + thread reopen, pi_context, @bot tags). Does not commit — the caller owns the + transaction. + """ + channel_id, visibility = await _resolve_channel(db, run_id, channel_name) + ts = mint_local_ts() + msg = AgentMessage( + simulation_run_id=run_id, + agent_id=None, # human/PI sender + channel_id=channel_id, + channel_name=channel_name, + message_ts=ts, + thread_ts=thread_ts, + phase="thread_reply" if thread_ts else "new_post", + visibility=visibility, + content=content, + sender_name=sender_name, + is_bot=False, + posted_at=float(ts), + ) + db.add(msg) + return msg + + +async def record_pi_dm( + db: AsyncSession, + *, + run_id: uuid.UUID, + agent_id: str, + pi_user_id: str, + direction: str, # 'inbound' (PI→bot) or 'outbound' (bot→PI) + content: str, + sender_name: str = "", + slack_ts: str | None = None, +) -> PiDmMessage: + """Persist a PI<->bot direct message. Does not commit.""" + ts = mint_local_ts() + dm = PiDmMessage( + simulation_run_id=run_id, + agent_id=agent_id, + pi_user_id=pi_user_id, + direction=direction, + content=content, + sender_name=sender_name, + ts=ts, + slack_ts=slack_ts, + posted_at=float(ts), + ) + db.add(dm) + return dm + + +def web_pi_user_id(user_id: uuid.UUID) -> str: + """Stable pi_user_id for a web (Slack-off) PI: ``local:``.""" + return f"local:{user_id}" diff --git a/src/services/private_channels.py b/src/services/private_channels.py index 76f8bd3..663efaa 100644 --- a/src/services/private_channels.py +++ b/src/services/private_channels.py @@ -29,6 +29,7 @@ from __future__ import annotations import logging +import time import uuid from dataclasses import dataclass @@ -36,20 +37,27 @@ from sqlalchemy.ext.asyncio import AsyncSession from src.agent.channels import normalize_channel_name -from src.agent.slack_client import AgentSlackClient +from src.agent.ids import mint_local_ts +from src.agent.slack_client import AgentSlackClient, ThreadNotFound from src.config import get_settings from src.models import ( + VISIBILITY_COLLAB_PRIVATE, + VISIBILITY_PUBLIC, AgentChannel, + AgentMessage, AgentRegistry, PrivateChannelMember, SimulationRun, ThreadDecision, User, - VISIBILITY_COLLAB_PRIVATE, ) logger = logging.getLogger(__name__) +# Neutral marker closing the origin public thread. Deliberately carries none of +# the PI's guidance text — that stays inside the private channel (§G6). +_CLOSE_MARKER_TEXT = "⏸️ continuing this discussion off-channel." + @dataclass class MigrationResult: @@ -83,6 +91,11 @@ def _build_slug(agent_a: str, agent_b: str, origin_channel_name: str) -> str: # under the limit with clean content boundaries. See observed split on # priv-lotz-su-single-cell-omics where a single ~4600-char handover landed # as two unrelated-looking posts (one orphaned mid-bullet). +# +# Kept below slack_client.SLACK_MAX_TEXT_CHARS deliberately: _add_handover_message +# writes ONE DB row per call, so a post that Slack splits would desynchronise the +# mirror (8515f65, defect 2). Pinned by +# tests/unit/test_slack_client_contract.py::test_handover_post_budget_stays_under_the_slack_split_threshold. _MAX_POST_CHARS = 3500 @@ -201,6 +214,82 @@ async def _latest_simulation_run_id(db: AsyncSession) -> uuid.UUID: return run_id +async def _slack_parent_ts_from_db( + db: AsyncSession, run_id: uuid.UUID, thread_ts: str, +) -> str | None: + """Resolve a canonical thread id to the Slack ts Slack must thread on. + + The DB-side twin of ``SimulationEngine._slack_parent_ts``: this process has no + MessageLog, so the root's mapping is read from ``agent_messages``. Returns None + when the thread has no Slack presence (a root minted while Slack was off), so + the caller can skip the mirror instead of posting against an id Slack has never + seen. ``slack_ts`` is the only evidence — a NULL means not on Slack; see + ``simulation._restored_slack_ts`` for why a missing mapping is no longer + inferred from the channel id. See specs/local-db-conversations.md. + """ + row = (await db.execute( + select(AgentMessage.slack_ts) + .where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.message_ts == thread_ts, + ) + .limit(1) + )).first() + if row is None: + # Root not stored at all (a run that predates content persistence). Fall + # back to the canonical id, preserving pure-Slack-on behaviour where the + # canonical id and the Slack ts are the same string. + return thread_ts + return row[0] + + +def _add_handover_message( + db: AsyncSession, + *, + simulation_run_id: uuid.UUID, + agent_id: str, + channel_id: str, + channel_name: str, + content: str, + visibility: str, + result: dict | None = None, + thread_ts: str | None = None, + slack_thread_ts: str | None = None, +) -> None: + """Record one handover message in ``agent_messages`` — the primary store. + + Used by both migration paths, so a handover exists in the DB whether or not + Slack is in play. The canonical id is the Slack ts when the mirror post landed, + else a locally-minted one — the same rule as ``SimulationEngine._post_message``, + which means a failed (or skipped) Slack post still leaves the message durable + and visible to the running simulation. The ``slack_*`` columns are only + populated when the post actually landed. See specs/local-db-conversations.md. + """ + slack_ts = (result or {}).get("ts") + ts = slack_ts or mint_local_ts() + db.add(AgentMessage( + simulation_run_id=simulation_run_id, + agent_id=agent_id, + channel_id=channel_id, + channel_name=channel_name, + message_ts=ts, + message_length=len(content), + thread_ts=thread_ts, + phase="thread_reply" if thread_ts else "new_post", + visibility=visibility, + content=content, + sender_name=f"{agent_id}Bot", + is_bot=True, + posted_at=float(ts), + slack_ts=slack_ts, + slack_channel_id=(result or {}).get("channel"), + # Only meaningful for a message that is itself on Slack, and it is the + # root's *Slack* ts — not the canonical thread_ts, which differ whenever + # the thread started Slack-off. + slack_thread_ts=slack_thread_ts if slack_ts else None, + )) + + async def _resolve_other_pi( db: AsyncSession, other_agent_id: str, ) -> tuple[AgentRegistry | None, User | None]: @@ -215,6 +304,106 @@ async def _resolve_other_pi( return reg, user +async def _slack_enabled_for_migration( + db: AsyncSession, creator_agent_id: str, other_agent_id: str, +) -> bool: + """Resolve whether the migration should use Slack. + + Explicit SLACK_ENABLED wins; otherwise auto-detect: Slack is used only when + both participating bots have usable tokens. See specs/local-db-conversations.md. + """ + settings = get_settings() + if settings.slack_enabled is not None: + return settings.slack_enabled + from src.services.slack_tokens import get_agent_bot_token + creator_tok = await get_agent_bot_token(db, creator_agent_id) + other_tok = await get_agent_bot_token(db, other_agent_id) + return bool(creator_tok and other_tok) + + +async def _migrate_offline( + db: AsyncSession, + *, + thread_decision: ThreadDecision, + creator_agent_id: str, + creator_pi_user: User, + guidance_text: str, + a: str, + b: str, + other_agent_id: str, + origin_channel_name: str, +) -> MigrationResult: + """Slack-off migration: DB-only, no Slack calls. + + Creates the collab_private AgentChannel and members exactly as the Slack + path, but with a local: channel id, and writes the handover posts and the + origin-thread ⏸️ close marker as agent_messages rows so the running sim (and + the next rebuild) pick them up through _poll_inbound_from_db. + """ + base_slug = _build_slug(a, b, origin_channel_name) + stamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + new_channel_name = normalize_channel_name(f"{base_slug[: 80 - len(stamp) - 1]}-{stamp}") + new_channel_id = f"local:{new_channel_name}" + origin_channel_id = f"local:{origin_channel_name}" + + simulation_run_id = await _latest_simulation_run_id(db) + + ac = AgentChannel( + simulation_run_id=simulation_run_id, + channel_id=new_channel_id, + channel_name=new_channel_name, + channel_type="collaboration", + visibility=VISIBILITY_COLLAB_PRIVATE, + created_by_agent=creator_agent_id, + migrated_from_channel_id=origin_channel_id, + ) + db.add(ac) + await db.flush() + + db.add(PrivateChannelMember(agent_channel_id=ac.id, agent_id=creator_agent_id, role="bot")) + db.add(PrivateChannelMember(agent_channel_id=ac.id, agent_id=other_agent_id, role="bot")) + db.add(PrivateChannelMember( + agent_channel_id=ac.id, user_id=creator_pi_user.id, role="pi", + added_by_user_id=creator_pi_user.id, + )) + + # Handover posts, authored by the creator bot, written straight to the DB + # (visibility=collab_private) so they stay within the channel's membership. + handover_posts = _build_handover_messages( + creator_pi_name=creator_pi_user.name, + proposal_summary=thread_decision.summary_text, + guidance_text=guidance_text, + origin_channel_name=origin_channel_name, + ) + # No Slack post to mirror, so _add_handover_message mints each canonical id + # from the process-wide minter (never a hand-rolled f"{time.time():.6f}": that + # carries no writer slot, so it can collide with an id minted by the engine or + # GrantBot, and it round-trips microseconds through a float, which cannot hold + # them at current epoch magnitudes — see src/agent/ids.py). + for post in handover_posts: + _add_handover_message( + db, simulation_run_id=simulation_run_id, agent_id=creator_agent_id, + channel_id=new_channel_id, channel_name=new_channel_name, + content=post, visibility=VISIBILITY_COLLAB_PRIVATE, + ) + # Neutral close marker in the origin (public) thread — no PI text echoed. + _add_handover_message( + db, simulation_run_id=simulation_run_id, agent_id=creator_agent_id, + channel_id=origin_channel_id, channel_name=origin_channel_name, + content=_CLOSE_MARKER_TEXT, visibility=VISIBILITY_PUBLIC, + thread_ts=thread_decision.thread_id, + ) + + thread_decision.refined_in_channel = new_channel_id + logger.info("Slack-off migration: created private channel %s (DB-only)", new_channel_name) + return MigrationResult( + channel_id=new_channel_id, + channel_name=new_channel_name, + agent_channel_id=ac.id, + invited_other_pi=False, + ) + + async def migrate_public_thread_to_private( db: AsyncSession, *, @@ -243,6 +432,25 @@ async def migrate_public_thread_to_private( origin_channel_name = thread_decision.channel + # Slack-off: DB-only migration (no channel/invite/post/DM). The handover is + # written straight to agent_messages for the sim to ingest. + if not await _slack_enabled_for_migration(db, creator_agent_id, other_agent_id): + return await _migrate_offline( + db, + thread_decision=thread_decision, + creator_agent_id=creator_agent_id, + creator_pi_user=creator_pi_user, + guidance_text=guidance_text, + a=a, b=b, + other_agent_id=other_agent_id, + origin_channel_name=origin_channel_name, + ) + + # Resolved up front, before any Slack side-effect: the handover messages are + # recorded against this run, and failing here *after* creating the Slack + # channel would leave an orphan channel behind. + simulation_run_id = await _latest_simulation_run_id(db) + # --- Slack side-effects ------------------------------------------------ creator_token = await _get_or_fail_bot_token(db, creator_agent_id) other_token = await _get_or_fail_bot_token(db, other_agent_id) @@ -290,15 +498,42 @@ async def migrate_public_thread_to_private( guidance_text=guidance_text, origin_channel_name=origin_channel_name, ) - for post in handover_posts: - creator_client.post_message(new_channel_id, post) + # Each Slack result is kept so the DB rows below can carry the canonical id + # Slack assigned (and the mirror mapping). The DB is the primary conversation + # store — a handover that existed only on Slack would be invisible to a + # Slack-off restart and to the web conversation view. + handover_results: list[tuple[str, dict | None]] = [ + (post, creator_client.post_message(new_channel_id, post)) + for post in handover_posts + ] - # Close the origin thread with a neutral marker — NO PI text echoed. - creator_client.post_message( - origin_channel_id, - "⏸️ continuing this discussion off-channel.", - thread_ts=thread_decision.thread_id, + # Close the origin thread with a neutral marker — NO PI text echoed. Slack + # threads on the root's *Slack* ts, which equals the canonical thread id only + # when the root was born on Slack, so translate first and keep the marker + # DB-only when the thread has no Slack presence. + slack_parent = await _slack_parent_ts_from_db( + db, simulation_run_id, thread_decision.thread_id, ) + close_result = None + if slack_parent is None: + logger.warning( + "Not mirroring the close marker for thread %s: its root has no Slack " + "presence (started with Slack off). The marker is still recorded in the DB.", + thread_decision.thread_id, + ) + else: + try: + close_result = creator_client.post_message( + origin_channel_id, _CLOSE_MARKER_TEXT, thread_ts=slack_parent, + ) + except ThreadNotFound: + # Origin root was deleted on Slack. post_message already cleaned up the + # orphan top-level post; the DB marker below still records the close, so + # the migration completes rather than aborting a PI-initiated action. + logger.warning( + "Origin thread %s no longer exists on Slack — close marker recorded " + "in the DB only", thread_decision.thread_id, + ) # Invite the other PI via DM from their own bot. Best-effort — if this # fails (no claimed PI, no Slack ID, DM not allowed), refinement still @@ -324,7 +559,6 @@ async def migrate_public_thread_to_private( ) # --- DB writes --------------------------------------------------------- - simulation_run_id = await _latest_simulation_run_id(db) ac = AgentChannel( simulation_run_id=simulation_run_id, channel_id=new_channel_id, @@ -354,6 +588,24 @@ async def migrate_public_thread_to_private( # The other PI is deliberately not added as a member here — they only # become a member when they accept the Slack invite. No DB write until then. + # Mirror the handover into agent_messages. Same rows as the Slack-off path, + # additionally carrying the slack_* mapping, so the running simulation picks + # them up via _poll_inbound_from_db and a rebuild reconstructs the channel + # from the DB alone rather than depending on Slack history. + for post, result in handover_results: + _add_handover_message( + db, simulation_run_id=simulation_run_id, agent_id=creator_agent_id, + channel_id=new_channel_id, channel_name=new_channel_name, + content=post, visibility=VISIBILITY_COLLAB_PRIVATE, result=result, + ) + _add_handover_message( + db, simulation_run_id=simulation_run_id, agent_id=creator_agent_id, + channel_id=origin_channel_id, channel_name=origin_channel_name, + content=_CLOSE_MARKER_TEXT, visibility=VISIBILITY_PUBLIC, + result=close_result, thread_ts=thread_decision.thread_id, + slack_thread_ts=slack_parent, + ) + # Record the refinement destination on the thread_decision thread_decision.refined_in_channel = new_channel_id diff --git a/src/services/profile_pipeline.py b/src/services/profile_pipeline.py index ec4bd44..1b4f543 100644 --- a/src/services/profile_pipeline.py +++ b/src/services/profile_pipeline.py @@ -9,14 +9,14 @@ 6. Prepare profile record 7. LLM synthesis (public profile) 8. Validation -9. Store + seed private profile (first creation only) +9. Store, gated on validation and recorded on the profile row (migration 0023), + + seed private profile (first creation only) """ import hashlib -import json import logging import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from sqlalchemy import select @@ -100,11 +100,16 @@ def update_progress(step: str, detail: str = ""): # Step 3: Fetch ORCID works update_progress("step3", "Fetching publication list from ORCID...") + # When this lookup FAILS we do not know how many works the researcher has, so + # step 9 must not record "0 identifiers" — that reads as "nothing to fetch" + # (a genuinely publication-less researcher) when it means "we could not ask". + works_lookup_failed = False try: orcid_works = await fetch_orcid_works(orcid_id) except Exception as exc: logger.warning("Step 3 failed: %s", exc) orcid_works = [] + works_lookup_failed = True # Extract PMIDs for works that have them pmids = [w["pmid"] for w in orcid_works if w.get("pmid")] @@ -323,20 +328,131 @@ def update_progress(step: str, detail: str = ""): except Exception as exc: logger.error("Retry synthesis failed: %s", exc) - # Step 9: Store + # Step 9: Store. + # + # `validated` is READ here. It used to gate only the retry above: step 9 stored + # on `if synthesized:` alone, so the retry's validation result was computed and + # thrown away, and a profile that failed _validate_profile twice was persisted + # as though it had passed. Nothing recorded the difference, so no test could + # see it — hardwiring _validate_profile to `return True` changed no observable + # behaviour at all. Two columns now record the decision (migration 0023): + # `synthesis_validated`, and the evidence counts that say what the stored + # fields are grounded in. + # + # The failure mode on a double validation failure is deliberate: store the + # draft and MARK it, rather than raise or store nothing. + # * Raising is loud in the log and silent in the UI. execute_generate_profile + # lets the exception reach process_job, which retries up to + # Job.max_attempts (default 3) — three more full LLM+NCBI runs for a + # formatting miss the retry above already tried to fix — and then sets + # status='dead'. templates/onboarding/profile_review.html keys its "Try + # Again" control on job_status == 'failed', which src/worker/main.py never + # assigns (it only ever writes 'pending' or 'dead'), so a dead job falls + # through to that template's `elif profile` branch and the PI is shown the + # review form with empty fields and no explanation. Raising would also + # skip step 9b, the markdown export and create_revision below, costing the + # private-profile seed and the audit trail. + # * Storing nothing is indistinguishable from "the pipeline never ran" and + # throws away the only draft the PI has to edit. (It would not cause the + # /onboarding re-enqueue loop: that self-heal is gated on `job is None and + # profile is None`, and step 6 above always creates the row first.) + # * Storing + marking keeps onboarding moving — the PI edits the draft and + # POSTs /onboarding/save-profile — while being distinguishable (one column, + # one ERROR log, one job-progress entry) and recoverable (POST + # /onboarding/retry, or the next monthly_refresh). + # + # What it will NOT do is let a worse synthesis overwrite a better stored one. + # A monthly refresh that fails validation, or one that runs while PubMed is + # down, keeps the profile that is already there. update_progress("step9", "Saving profile to database...") profile.grant_titles = grant_titles or profile.grant_titles + # Records this run's INPUT (change detection), so it is written even when the + # synthesized fields below are not. The evidence counts are the ones that + # describe the stored profile. profile.raw_abstracts_hash = abstracts_hash + # What the pipeline should have been able to fetch, and what actually reached + # the prompt. Both zero means there was nothing to fetch; the first non-zero + # with the second zero means the fetch failed and whatever the model wrote is + # ungrounded. None for the first means step 3 could not even enumerate the + # works, so "nothing to fetch" cannot be claimed. See + # ResearcherProfile.evidence_state. + evidence_pmid_count = None if works_lookup_failed else len(set(pmids)) + evidence_pub_count = len(pubs_for_synthesis) + if synthesized: - profile.research_summary = synthesized.get("research_summary", "") - profile.techniques = synthesized.get("techniques", []) - profile.experimental_models = synthesized.get("experimental_models", []) - profile.disease_areas = synthesized.get("disease_areas", []) - profile.key_targets = synthesized.get("key_targets", []) - profile.keywords = synthesized.get("keywords", []) - profile.profile_version = (profile.profile_version or 0) + 1 - profile.profile_generated_at = datetime.now(timezone.utc) + stored_is_worth_keeping = ( + (profile.profile_version or 0) > 0 + and bool(profile.research_summary) + # A stored profile already known to have failed validation is not + # worth protecting. NULL (legacy/unknown) is. + and profile.synthesis_validated is not False + ) + lost_evidence = evidence_pub_count == 0 and (profile.evidence_pub_count or 0) > 0 + if stored_is_worth_keeping and (not validated or lost_evidence): + reason = ( + "failed validation twice" + if not validated + else f"grounded in 0 publications, down from {profile.evidence_pub_count}" + ) + logger.error( + "Discarding synthesized profile for %s (%s); keeping stored version %d", + user.name, reason, profile.profile_version, + ) + update_progress( + "validation_rejected", + f"Kept the existing profile (version {profile.profile_version}): " + f"the new synthesis {reason}.", + ) + else: + profile.research_summary = synthesized.get("research_summary", "") + profile.techniques = synthesized.get("techniques", []) + profile.experimental_models = synthesized.get("experimental_models", []) + profile.disease_areas = synthesized.get("disease_areas", []) + profile.key_targets = synthesized.get("key_targets", []) + profile.keywords = synthesized.get("keywords", []) + profile.synthesis_validated = validated + profile.evidence_pmid_count = evidence_pmid_count + profile.evidence_pub_count = evidence_pub_count + profile.profile_version = (profile.profile_version or 0) + 1 + profile.profile_generated_at = datetime.now(UTC) + + if not validated: + logger.error( + "Stored an UNVALIDATED profile for %s (version %d): failed " + "_validate_profile on both attempts. Marked " + "synthesis_validated=False for regeneration.", + user.name, profile.profile_version, + ) + update_progress( + "unvalidated", + "The generated profile did not meet the quality checks " + "(150-250 word summary, 3+ techniques, 1+ disease area). " + "It was saved as a draft for you to edit.", + ) + if evidence_pub_count == 0: + # Nothing the researcher wrote reached the prompt, so whatever the + # model produced came from its own priors plus a name and a + # department. It is stored (a PubMed outage must not stop a PI + # being onboarded, and some researchers really have no indexed + # papers) but it is no longer indistinguishable from a real one. + found = ( + "an unknown number of" + if evidence_pmid_count is None + else str(evidence_pmid_count) + ) + logger.error( + "Stored an UNGROUNDED profile for %s: 0 publication abstracts " + "reached the synthesis prompt (%s publication IDs in hand, " + "evidence_state=%s)", + user.name, found, profile.evidence_state, + ) + update_progress( + "ungrounded", + f"No publication abstracts reached the profile synthesis " + f"({found} publication IDs were found): " + f"{profile.evidence_state}.", + ) # Step 9b: Generate private profile seed (if no live profile and no existing seed) if not profile.private_profile_md and not profile.private_profile_seed: diff --git a/src/services/profile_versioning.py b/src/services/profile_versioning.py index c02b5aa..f00172c 100644 --- a/src/services/profile_versioning.py +++ b/src/services/profile_versioning.py @@ -15,6 +15,36 @@ logger = logging.getLogger(__name__) +async def latest_revision( + db: AsyncSession, + *, + agent_registry_id: uuid.UUID, + profile_type: str, +) -> ProfileRevision | None: + """The newest revision for one (agent, profile_type), or None if there is none. + + Backed by ``ix_profile_revision_agent_type_created``, which is already ordered + ``created_at DESC``. + + Note on ties: ``created_at`` defaults to ``now()``, which in Postgres is the + *transaction* timestamp, so two revisions for the same key written inside one + transaction share it and "newest" is then arbitrary between them. Every caller + writes at most one revision per key per transaction, so this does not arise in + practice; a caller that needs to write several must commit between them. + """ + return ( + await db.execute( + select(ProfileRevision) + .where( + ProfileRevision.agent_registry_id == agent_registry_id, + ProfileRevision.profile_type == profile_type, + ) + .order_by(ProfileRevision.created_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + + async def create_revision( db: AsyncSession, *, @@ -25,7 +55,7 @@ async def create_revision( mechanism: str, change_summary: str | None = None, ) -> ProfileRevision: - """Create a profile revision record. + """Create a profile revision record, unless it would repeat the previous one. Args: db: Database session. @@ -37,8 +67,30 @@ async def create_revision( change_summary: Optional short description of what changed. Returns: - The created ProfileRevision. + The created ProfileRevision, or the existing newest one when ``content`` is + byte-identical to it — see below. """ + # A revision whose content repeats its predecessor's is not history, it is noise: + # it records that something was written, not that anything changed. This appended + # unconditionally, so re-running `backfill-profile-revisions` doubled every row + # with a byte-identical twin. Keyed on content alone, deliberately — a differing + # `mechanism` or `change_summary` over the same body is still the same profile, + # and the point of the history is what the profile said. + # + # Only the *newest* revision is compared, so a genuine edit always lands, and so + # does a revert back to an older body. See the two halves pinned in + # tests/integration/test_cli.py: test_backfill_run_twice_does_not_duplicate_any_ + # revision and test_a_changed_profile_body_still_creates_a_new_revision. + previous = await latest_revision( + db, agent_registry_id=agent_registry_id, profile_type=profile_type + ) + if previous is not None and previous.content == content: + logger.debug( + "Skipped unchanged %s profile revision for agent %s (via %s)", + profile_type, agent_registry_id, mechanism, + ) + return previous + revision = ProfileRevision( agent_registry_id=agent_registry_id, profile_type=profile_type, diff --git a/src/services/pubmed.py b/src/services/pubmed.py index 24456dd..8a8b2fd 100644 --- a/src/services/pubmed.py +++ b/src/services/pubmed.py @@ -73,11 +73,21 @@ def reconcile_pub_doi( _request_semaphore = asyncio.Semaphore(8) # Conservative limit +# NCBI's E-utilities usage policy requires every request to identify the caller with +# `tool` and `email`. Anonymous traffic is throttled first and IP-blocked second, and +# NCBI has no way to warn us because it does not know who we are. Every NCBI call in +# the system — the profile pipeline, DOI reconciliation, PMC methods extraction — +# funnels through _ncbi_get, so omitting these made the whole deployment anonymous. +_NCBI_TOOL = "copi-science" + + async def _ncbi_get(url: str, params: dict[str, Any]) -> httpx.Response: - """Make a rate-limited GET request to NCBI.""" + """Make a rate-limited, identified GET request to NCBI.""" settings = get_settings() if settings.ncbi_api_key: params["api_key"] = settings.ncbi_api_key + params.setdefault("tool", _NCBI_TOOL) + params.setdefault("email", settings.ncbi_contact_email or settings.ses_sender_email) async with _request_semaphore: async with httpx.AsyncClient(timeout=60, follow_redirects=True) as client: resp = await client.get(url, params=params) diff --git a/src/services/slack_provisioning.py b/src/services/slack_provisioning.py index 055cb58..f1c0b6c 100644 --- a/src/services/slack_provisioning.py +++ b/src/services/slack_provisioning.py @@ -28,6 +28,12 @@ "chat:write", # chat.postMessage "groups:history", # threads in private channels "groups:read", # conversations.list private + # conversations.create(is_private=True) and conversations.invite into a private + # channel both require this. Without it a bot provisions, connects and posts + # perfectly, and then private-channel migration — the PI-pairing feature — fails + # with missing_scope. Adding a scope needs every existing bot REINSTALLED; an + # already-installed app keeps the grant it was installed with. + "groups:write", # conversations.create/invite for private channels "im:history", # poll_dm_messages "im:write", # conversations.open (DMs) "users:read", # users.info @@ -75,13 +81,25 @@ def create_app( pi_name: str, redirect_uri: str, max_rate_limit_retries: int = 5, + scopes: list[str] | None = None, ) -> dict: """Create one Slack app via the Manifest API. Returns a dict with ``agent_id``, ``bot_name``, ``pi_name``, ``app_id``, ``client_id``, ``client_secret``, ``oauth_url``. Retries on rate-limit responses only; all other errors raise immediately. + + ``scopes`` defaults to ``BOT_SCOPES``. It is overridable because the scope set + is fixed at *manifest* time and cannot be changed at install time — Slack's + consent screen offers Allow or Cancel, not a per-scope choice, and an + already-installed app keeps the grant it was installed with. The live Slack + tier depends on that: ``wiseman`` is the control that must NOT hold + ``groups:write`` (see + ``test_slack_provision_live.py::test_the_granted_scopes_are_the_scopes_we_asked_for`` + and ``test_private_channel_creation_needs_groups_write``), so it has to be + created from a reduced manifest or the asymmetry is unreproducible. """ + scopes = list(BOT_SCOPES) if scopes is None else list(scopes) manifest = { "display_information": { "name": bot_name, @@ -95,7 +113,7 @@ def create_app( }, "oauth_config": { "redirect_urls": [redirect_uri], - "scopes": {"bot": BOT_SCOPES}, + "scopes": {"bot": scopes}, }, "settings": { "org_deploy_enabled": False, diff --git a/src/services/slack_tokens.py b/src/services/slack_tokens.py index d226db7..05547c9 100644 --- a/src/services/slack_tokens.py +++ b/src/services/slack_tokens.py @@ -18,7 +18,23 @@ def is_valid_token(token: str | None) -> bool: - return bool(token) and not token.startswith("xoxb-placeholder") + """True for a value that could plausibly be a Slack **bot** token. + + Every caller passes a bot token, and Slack bot tokens are always ``xoxb-``. The + prefix check is not cosmetic: ``slack_globally_enabled()`` auto-detects Slack as ON + from the mere *presence* of a valid-looking token, so whatever this accepts is what + can switch the whole integration on. Before the prefix check, a user token + (``xoxp-``), an app-config token (``xoxe.xoxp-`` — which lives in the same ``.env`` + as the bot tokens), a stray ``" "``, or an unfilled ``REPLACE_ME`` all counted as + "usable", flipping Slack on and then failing every API call with ``invalid_auth`` + or ``not_allowed_token_type``. + + ``xoxb-placeholder`` remains a recognised no-op value for seeded rows. + """ + if not token: + return False + token = token.strip() + return token.startswith("xoxb-") and not token.startswith("xoxb-placeholder") def env_token(agent_id: str) -> str | None: @@ -49,6 +65,20 @@ async def get_agent_bot_token(db: AsyncSession, agent_id: str) -> str | None: return env_token(agent_id) +async def slack_globally_enabled(db: AsyncSession) -> bool: + """Whether Slack integration is on for this deployment. + + Explicit SLACK_ENABLED wins; otherwise auto-detect (on iff at least one + usable bot token exists anywhere). Used to gate secondary Slack posters + (GrantBot, the email→Slack relay, web-triggered posts) so they no-op in + DB-only mode. See specs/local-db-conversations.md. + """ + setting = get_settings().slack_enabled + if setting is not None: + return setting + return await get_any_bot_token(db) is not None + + async def get_any_bot_token(db: AsyncSession) -> str | None: """Any valid bot token, for workspace-wide lookups (e.g. users.lookupByEmail). diff --git a/src/services/slack_web.py b/src/services/slack_web.py new file mode 100644 index 0000000..832c1cc --- /dev/null +++ b/src/services/slack_web.py @@ -0,0 +1,300 @@ +"""The Slack boundary for the web and service layers. + +`src/agent/slack_client.py` is the chokepoint for the simulation engine. It was +built because pagination, retry and message splitting had each been reimplemented +— or forgotten — per call site, and four defects turned out to be four instances +of one structural absence. That reasoning applies identically outside the engine, +where eight call sites had constructed `slack_sdk.WebClient` directly: two read a +single 200-item page of a paginated endpoint, none retried a 429, and one posted +unsplit bodies that Slack silently chunked. + +This module is the second half of that boundary. `tests/unit/test_slack_boundary.py` +asserts that `slack_sdk` is imported in exactly two modules, so a ninth bypass is +a failing test rather than a defect discovered in production. + +The core is synchronous, because ``slack_sdk.WebClient`` is and because GrantBot +and one route helper have no event loop. **Async callers must use the ``_async`` +wrappers at the bottom of this module, not the sync functions.** Six of the seven +call sites are FastAPI route handlers, and a synchronous ``time.sleep`` inside one +of those stalls the whole event loop, not just that request — see ``_call``. +""" +from __future__ import annotations + +import asyncio +import logging +import time +from typing import Any + +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + +from src.agent.slack_client import ( + MAX_PAGES, + SLACK_MAX_TEXT_CHARS, + SLACK_PAGE_LIMIT, + SlackListingIncomplete, + split_for_slack, +) + +logger = logging.getLogger(__name__) + +_MAX_ATTEMPTS = 4 +_BACKOFF_BASE = 0.5 +# Slack can answer a 429 with Retry-After in the tens of seconds. Honouring it +# exactly is right for not getting throttled harder, but three of them would hold a +# request for minutes, so it is capped and the cap is logged. The cap is only safe +# because async callers reach this through the _async wrappers, which run it in a +# worker thread — a bare time.sleep on a route handler's own thread would block +# every other request in the process, not just this one. +_MAX_RETRY_AFTER = 30.0 + +# Errors that mean "this call will never work", so retrying is pointless. +# ``user_not_found`` is users.info's spelling and ``users_not_found`` is +# users.lookupByEmail's; both are here because a user who does not exist does not +# start existing on attempt four, and the callers that translate them to None sit +# in synchronous request paths where four attempts costs 3.5s of backoff. +_TERMINAL = frozenset({ + "invalid_auth", "account_inactive", "token_revoked", "no_permission", + "user_not_found", "users_not_found", "channel_not_found", "not_in_channel", +}) + +__all__ = [ + "SlackListingIncomplete", + "get_user_info", + "get_user_info_async", + "join_channel", + "join_channel_async", + "list_channel_ids", + "list_channel_ids_async", + "lookup_user_by_email", + "lookup_user_by_email_async", + "post_message", + "post_message_async", +] + + +def _client(token: str) -> WebClient: + """Seam for tests; the only WebClient construction in the web layer.""" + return WebClient(token=token) + + +def _error_code(exc: SlackApiError) -> str: + """Slack's ``error`` string for a failed call, or ``""`` when it sent none.""" + return (exc.response.get("error") if exc.response else None) or "" + + +def _call(client: WebClient, method: str, **kwargs: Any) -> Any: + """One Slack call with bounded retry on rate limits and transient errors. + + Honours ``Retry-After`` when Slack sends it, because guessing is how a + throttled bot becomes a blocked bot. Terminal errors raise immediately: a + revoked token does not become valid on attempt four. + """ + last: Exception | None = None + for attempt in range(_MAX_ATTEMPTS): + try: + return getattr(client, method)(**kwargs) + except SlackApiError as exc: + code = _error_code(exc) + if code in _TERMINAL: + raise + last = exc + if attempt == _MAX_ATTEMPTS - 1: + break + delay = _BACKOFF_BASE * (2 ** attempt) + if code == "ratelimited": + retry_after = (getattr(exc.response, "headers", {}) or {}).get("Retry-After") + if retry_after is not None: + try: + asked = float(retry_after) + except (TypeError, ValueError): + asked = delay + if asked > _MAX_RETRY_AFTER: + logger.warning( + "[slack_web] %s asked for Retry-After=%.0fs; capping at %.0fs", + method, asked, _MAX_RETRY_AFTER, + ) + delay = min(asked, _MAX_RETRY_AFTER) + logger.warning("[slack_web] %s failed (%s); retrying in %.1fs", method, code, delay) + if delay > 0: + time.sleep(delay) + assert last is not None + raise last + + +def list_channel_ids( + token: str, + *, + include_private: bool = True, + exclude_archived: bool = False, +) -> dict[str, str]: + """Every channel the token can see, as ``{name: id}``. Fully paginated. + + Raises ``SlackListingIncomplete`` carrying ``.partial`` rather than returning + a subset that looks whole — a subset is what makes a caller conclude a channel + does not exist when it is merely on page two. + + ``exclude_archived`` defaults to False because the callers that ask "does this + name exist" must count archived channels: an archived channel still owns its + name. Pass True when the answer feeds an action that archived channels cannot + take, such as joining. + """ + types = "public_channel,private_channel" if include_private else "public_channel" + out: dict[str, str] = {} + cursor = "" + seen: set[str] = set() + client = _client(token) + + for page in range(MAX_PAGES): + call: dict[str, Any] = { + "types": types, + "limit": SLACK_PAGE_LIMIT, + "exclude_archived": exclude_archived, + } + if cursor: + call["cursor"] = cursor + try: + result = _call(client, "conversations_list", **call) + except SlackApiError as exc: + if page == 0: + raise + raise SlackListingIncomplete( + "conversations.list", out, + f"page {page + 1} failed: {_error_code(exc) or exc}", + ) from exc + + for ch in result.get("channels") or []: + out[ch["name"]] = ch["id"] + + cursor = ((result.get("response_metadata") or {}).get("next_cursor") or "").strip() + if not cursor: + return out + if cursor in seen: + raise SlackListingIncomplete( + "conversations.list", out, f"Slack repeated cursor {cursor!r}") + seen.add(cursor) + + raise SlackListingIncomplete("conversations.list", out, f"exceeded {MAX_PAGES} pages") + + +def lookup_user_by_email(token: str, email: str) -> str | None: + """Slack user id for an email, or None when Slack has no such user.""" + try: + result = _call(_client(token), "users_lookupByEmail", email=email) + except SlackApiError as exc: + if _error_code(exc) == "users_not_found": + return None + raise + return ((result.get("user") or {}).get("id")) or None + + +def get_user_info(token: str, user_id: str) -> dict[str, Any] | None: + """The ``user`` object for a Slack id, or None when it does not resolve.""" + try: + result = _call(_client(token), "users_info", user=user_id) + except SlackApiError as exc: + if _error_code(exc) in {"user_not_found", "users_not_found"}: + return None + raise + return result.get("user") or None + + +def join_channel(token: str, channel_id: str) -> None: + """Join a channel. ``already_in_channel`` is success, not failure.""" + try: + _call(_client(token), "conversations_join", channel=channel_id) + except SlackApiError as exc: + if _error_code(exc) == "already_in_channel": + return + raise + + +def post_message( + token: str, + channel: str, + text: str, + *, + thread_ts: str | None = None, +) -> list[dict[str, Any]]: + """Post ``text``, split so no chunk exceeds Slack's limit. + + Returns one record per Slack message actually created. Callers that persist + what they posted must write one row per returned record, or the DB and Slack + disagree about how many messages exist — measured live at >4000 characters, + where Slack silently splits and returns only the last ts. + + ``thread_ts`` exists because two callers post *threaded* replies — the + legacy PI-guidance path in ``routers/agent_page.py`` and its email + equivalent in ``services/email_inbound.py``. Without it they could not come + through here at all: posting their guidance without a ``thread_ts`` would + move it out of the proposal thread and into the channel root, which is a + worse defect than the raw client they were using. It is omitted from the + payload entirely when None, so a top-level post is byte-identical to before + this parameter existed. + """ + client = _client(token) + posted: list[dict[str, Any]] = [] + for chunk in split_for_slack(text, SLACK_MAX_TEXT_CHARS): + call: dict[str, Any] = {"channel": channel, "text": chunk} + if thread_ts: + call["thread_ts"] = thread_ts + result = _call(client, "chat_postMessage", **call) + posted.append({ + "ts": result.get("ts"), + "channel": channel, + "text": chunk, + "thread_ts": thread_ts, + }) + return posted + + +# --------------------------------------------------------------------------- +# Async wrappers — the entry point for every FastAPI route handler. +# +# The sync functions above call slack_sdk, which blocks on network I/O, and _call +# adds up to three time.sleep()s on top of that. Called directly from an `async +# def` route those block the event loop, so ONE throttled Slack call freezes every +# other request the process is serving. That is strictly worse than the raw +# WebClient these functions replaced: it had no retry, so its worst case was a +# single blocking HTTP call rather than four plus backoff. +# +# asyncio.to_thread moves the whole thing to a worker thread, so the wait costs +# that request its latency and nothing else. Six of the seven call sites are async; +# GrantBot and _resolve_delegate_names are sync and use the plain functions. +# --------------------------------------------------------------------------- + + +async def list_channel_ids_async( + token: str, + *, + include_private: bool = True, + exclude_archived: bool = False, +) -> dict[str, str]: + """``list_channel_ids`` off the event loop.""" + return await asyncio.to_thread( + list_channel_ids, token, + include_private=include_private, exclude_archived=exclude_archived, + ) + + +async def lookup_user_by_email_async(token: str, email: str) -> str | None: + """``lookup_user_by_email`` off the event loop.""" + return await asyncio.to_thread(lookup_user_by_email, token, email) + + +async def get_user_info_async(token: str, user_id: str) -> dict[str, Any] | None: + """``get_user_info`` off the event loop.""" + return await asyncio.to_thread(get_user_info, token, user_id) + + +async def join_channel_async(token: str, channel_id: str) -> None: + """``join_channel`` off the event loop.""" + return await asyncio.to_thread(join_channel, token, channel_id) + + +async def post_message_async( + token: str, channel: str, text: str, *, thread_ts: str | None = None +) -> list[dict[str, Any]]: + """``post_message`` off the event loop.""" + return await asyncio.to_thread( + post_message, token, channel, text, thread_ts=thread_ts) diff --git a/src/visibility.py b/src/visibility.py new file mode 100644 index 0000000..565bad2 --- /dev/null +++ b/src/visibility.py @@ -0,0 +1,20 @@ +"""Channel visibility vocabulary — dependency-free. + +Lives outside ``src/models`` so modules that must stay free of DB/ORM imports can +still speak the vocabulary. ``src.models.agent_activity`` re-exports both names, so +every existing importer keeps working unchanged. + +See specs/privacy-and-channel-visibility.md: +- ``public`` — all bots and PIs; seeded and agent-created thematic channels. +- ``collab_private`` — 2 bots + up to 2 PIs; Slack ``is_private=true``. + +``src/agent/message_log.py`` needs ``VISIBILITY_COLLAB_PRIVATE`` for the cohort +gate's private-channel exemption (.notes/cohort-system-v2.md §7) and is otherwise +dependency-free; importing the ORM module there would couple the in-memory log to +SQLAlchemy for the sake of one string. +""" + +VISIBILITY_PUBLIC = "public" +VISIBILITY_COLLAB_PRIVATE = "collab_private" + +__all__ = ["VISIBILITY_PUBLIC", "VISIBILITY_COLLAB_PRIVATE"] diff --git a/templates/admin/_cohort_gate_banner.html b/templates/admin/_cohort_gate_banner.html new file mode 100644 index 0000000..e5e3d5a --- /dev/null +++ b/templates/admin/_cohort_gate_banner.html @@ -0,0 +1,98 @@ +{# + Shared cohort-gate status banner. Included by cohorts.html, cohort_detail.html + and cohort_topology.html so all three tell the same story about what is actually + in force. Expects the `gate` context dict built by _cohort_gate_context(). + See .notes/cohort-system-v2.md §12. +#} +{% if gate.preflight_error %} +
    +

    Cohort isolation is switched on but forced OFF.

    +

    {{ gate.preflight_error }}

    +

    + The simulation is running all-vs-all. Nothing has been silenced. +

    +
    +{% elif gate.isolation_enabled %} +
    +

    + Cohort isolation is ACTIVE + — policy {{ gate.default_policy }}. + {{ gate.summary.gated }} of {{ gate.summary.total }} active agents are gated; + {{ gate.summary.isolated | length }} isolated. +

    + {% if gate.summary.isolated %} +

    + Isolated (in no cohort, policy isolated): these agents act only + on human messages — + {% for aid in gate.summary.isolated %}{{ gate.bot_names.get(aid, aid) }}{% if not loop.last %}, {% endif %}{% endfor %} +

    + {% endif %} + {% if gate.default_policy == 'open' and gate.summary.unrestricted %} +

    + Unrestricted (in no cohort, policy open) — + {% for aid in gate.summary.unrestricted %}{{ gate.bot_names.get(aid, aid) }}{% if not loop.last %}, {% endif %}{% endfor %} +

    + {% endif %} +
    +{% else %} +
    + Cohort isolation is OFF + (cohort_isolation_enabled=false). Cohorts below are recorded but not + applied — the roster is all-vs-all. Topology edits are kept and take effect the + moment isolation is turned on; turning it on requires an agent-run + restart, because the setting is read once per process. +
    +{% endif %} + +
    +

    What the gate does and does not do

    +
      +
    • It filters what an agent acts on. It never changes what a human can read — PI and admin views always show the full history.
      • +
    • Human messages always pass, in every configuration.
      • +
    • PI-created private collaboration channels always pass: an explicit human pairing outranks a cohort.
      • +
    • A thread that is already open keeps getting replies so it can conclude, but it loses scheduling priority once its partner leaves the cohort.
      • +
    • Filtering is forward-only. Adding an agent to a cohort does not replay the messages it missed while excluded.
      • +
    • Membership edits are picked up by a running simulation within ~30 seconds. No restart.
      • +
    • But the settings are cached per process: changing + cohort_isolation_enabled or cohort_default_policy + requires restarting agent-run. Only the topology is live.
      • +
    +
    + +{% if gate.snapshot and gate.snapshot.topology %} +{% set snap = gate.snapshot.topology %} +{% set counters = snap.get('counters') or {} %} +
    + + Last topology recorded by a running simulation + + {{ gate.snapshot.created_at.strftime('%b %d %H:%M') }} + + +
    +

    + Written by the engine process, so these are the only numbers this page can + show about the gate's real effect. + Isolation was {{ snap.get('cohort_isolation_enabled') }}, + policy {{ snap.get('cohort_default_policy') }}, + reactive valve {{ snap.get('max_consecutive_reactive_turns') }}. +

    +

    + Cross-cohort mentions stripped: + {% if counters.get('tags_stripped') %} + {% for aid, n in counters['tags_stripped'].items() %}{{ aid }}={{ n }}{% if not loop.last %}, {% endif %}{% endfor %} + — a high rate means the topology disagrees with what the agents are trying to do. + {% else %}none{% endif %} +

    +

    + Grandfathered threads (partner left the cohort, thread allowed to conclude): + {{ counters.get('grandfathered_threads') | length if counters.get('grandfathered_threads') else 0 }} +

    +

    + Turn selection: {{ counters.get('reactive_selections', 0) }} reactive / + {{ counters.get('proactive_selections', 0) }} proactive +

    +
    +
    +{% endif %} diff --git a/templates/admin/activity_detail.html b/templates/admin/activity_detail.html index 10bd800..b8cab10 100644 --- a/templates/admin/activity_detail.html +++ b/templates/admin/activity_detail.html @@ -54,7 +54,7 @@

    Messages by Agent

     {% for agent_id, stats in agent_stats.items() | sort(attribute='1.count', reverse=True) %} - + @@ -124,7 +124,7 @@

    Message Timeline ({{ messages | len {% for msg in messages %}
    - {{ msg.agent_id }}Bot + {% if msg.agent_id %}{{ msg.agent_id }}Bot{% else %}(unknown sender){% endif %} #{{ msg.channel_name }} {{ msg.created_at.strftime('%b %d %H:%M:%S') }} ({{ msg.phase }}) diff --git a/templates/admin/agent_detail.html b/templates/admin/agent_detail.html index 0f581c3..7eea1cd 100644 --- a/templates/admin/agent_detail.html +++ b/templates/admin/agent_detail.html @@ -27,6 +27,11 @@

    ⚠️ Slack provisioning failed: {{ slack_error }}

    {% endif %} + {% if role_error %} +
    + ⚠️ {{ role_error }} +
    + {% endif %}
    @@ -43,6 +48,12 @@

    PI: {{ agent.pi_name }}

    +
    + Role: + + {{ agent.role }} + +
    {% if linked_user %}
    Linked user: @@ -118,6 +129,27 @@

    +
    +
    + + +

    + Selects the agent's per-role prompt overrides and tool allow-list + (prompts/roles/<role>/). pi_lab is the legacy + default. A running sim picks up the change on its next roster sync. +

    +
    + + + {% if agent.status == 'pending' %}

    + +
    +
    +

    {{ cohort.name }}

    + {% if cohort.description %}

    {{ cohort.description }}

    {% endif %} +
    +
    + + Topology matrix + + {% if cohort.memberships %} + {# The server refuses this too (v2 §12): deleting a populated cohort cascades + its memberships away and silently reshapes a running simulation. #} + + Delete Cohort + + {% else %} + + + + {% endif %} +
    +
    + +{% if error %} +
    {{ error }}
    +{% endif %} +{% if notice %} +
    {{ notice }}
    +{% endif %} + +{% include "admin/_cohort_gate_banner.html" %} + + +

    Members ({{ cohort.memberships | length }})

    +
    + {% if cohort.memberships %} +

    {{ agent_id | capitalize }}Bot{% if agent_id %}{{ agent_id | capitalize }}Bot{% else %}(unknown sender){% endif %} {{ stats.count }} {{ stats.avg_length }} chars
    + + + + + + + + + + + + + {% for m in cohort.memberships %} + {% set agent = agent_by_id.get(m.agent_id) %} + + + + + + + + + + {% endfor %} + +
    Agent IDBot NamePI NameStatusAdded byAddedActions
    {{ m.agent_id }}{{ agent.bot_name if agent else '—' }}{{ agent.pi_name if agent else '—' }} + {% if agent %} + {{ agent.status }} + {% else %} + unknown + {% endif %} + {{ adder_map.get(m.added_by | string, '—') }}{{ m.added_at.strftime('%b %d') }} +
    + + +
    +
    + {% else %} +

    No members yet. Add one below.

    + {% endif %} + + + +

    Add Agent

    +
    +
    + + +
    + +
    + + +

    Agent Cohort Map

    +

    All agents and the cohorts they currently belong to (read-only; manage membership from each cohort's page).

    +
    + + + + + + + + + {% for a in all_agents %} + + + + + {% endfor %} + +
    AgentCohorts
    {{ a.bot_name }} + {% set names = agent_cohort_map.get(a.agent_id) %} + {% if names %}{{ names | join(', ') }}{% else %}(none){% endif %} +
    +
    + + +

    Audit log

    +

    + Every change to this cohort, newest first. Append-only, and it outlives the + cohort row itself — deleting a cohort does not erase its history. +

    +
    + {% if audit_events %} + + + + + + + + + + + {% for e in audit_events %} + + + + + + + {% endfor %} + +
    WhenActionAgentBy
    + {{ e.created_at.strftime('%b %d %H:%M') }} + {{ e.action }} + {% if e.agent_id %}{{ agent_by_id[e.agent_id].bot_name if e.agent_id in agent_by_id else e.agent_id }}{% else %}—{% endif %} + {{ e.actor_email or '—' }}
    + {% else %} +

    No changes recorded yet.

    + {% endif %} +
    +{% endblock %} diff --git a/templates/admin/cohort_topology.html b/templates/admin/cohort_topology.html new file mode 100644 index 0000000..b95dd71 --- /dev/null +++ b/templates/admin/cohort_topology.html @@ -0,0 +1,138 @@ +{% extends "base.html" %} +{% block title %}Admin — Cohort topology — CoPI{% endblock %} + +{% block content %} +
    + ← All cohorts +
    + +
    +

    Topology matrix

    +
    +

    + Every agent × cohort pair. Tick to add a membership, untick to remove, then save + once — a whole re-shaping in a single audited step. Only the cells shown here are + compared, so a filtered or stale view can never delete a membership it did not + display. +

    + +{% if error %} +
    {{ error }}
    +{% endif %} +{% if notice %} +
    {{ notice }}
    +{% endif %} + +{% include "admin/_cohort_gate_banner.html" %} + +{% if not cohorts %} +
    +

    + No cohorts yet. Create one + before editing the topology. +

    +
    +{% elif not agents %} +
    +

    No agents in the registry yet.

    +
    +{% else %} +
    + {# Rendered rows and columns. The save reconstructs the rendered CELL set as + the cross product of these two lists, which is exactly what the template + renders below (an unconditional nested loop). Same stale-form guarantee as + the old per-cell `present` inputs — a column or row that was not displayed + cannot be diffed away — at 116 fields instead of 3,360. #} + {% for a in agents %}{% endfor %} + {% for c in cohorts %}{% endfor %} +
    + + + + + + + {% for c in cohorts %} + + {% endfor %} + + + + + {% for a in agents %} + {% set preview = gate.preview.get(a.agent_id, '__absent__') %} + + + + + {% for c in cohorts %} + {% set cell = c.id | string ~ ':' ~ a.agent_id %} + + {% endfor %} + + + {% endfor %} + +
    AgentStatusRole + {{ c.name }} +
    + +
    +     		Acts on (preview)
    + {{ a.bot_name }} +
    {{ a.pi_name }}
    +     		+ {{ a.status }} + + {{ a.role }} + + + + {% if a.status != 'active' %} + not active — the engine will not load this agent + {% elif preview == '__absent__' %} + — + {% elif preview is none %} + everyone (gate off for this agent) + {% elif not preview %} + humans + PI private channels only + {% else %} + {% for aid in preview if aid != a.agent_id %}{{ gate.bot_names.get(aid, aid) }}{% if not loop.last %}, {% endif %}{% else %}humans + PI private channels only{% endfor %} + {% endif %} +
    +
    + +
    + + Reset +

    + The “Acts on” column shows the gate as of page load; save to recompute. +

    +
    +
    + + +{% endif %} +{% endblock %} diff --git a/templates/admin/cohorts.html b/templates/admin/cohorts.html new file mode 100644 index 0000000..1aa44d2 --- /dev/null +++ b/templates/admin/cohorts.html @@ -0,0 +1,103 @@ +{% extends "base.html" %} +{% block title %}Admin — Cohorts — CoPI{% endblock %} + +{% block content %} +
    +

    Cohorts

    +
    + + Topology matrix + + +
    +
    +

    + A cohort groups agents permitted to act on each other's activity during + simulation. Use the topology + matrix to edit many memberships at once. +

    + +{% if error %} +
    {{ error }}
    +{% endif %} +{% if notice %} +
    {{ notice }}
    +{% endif %} + +{% include "admin/_cohort_gate_banner.html" %} + + + +
    + {% if cohorts %} + + + + + + + + + + + + + + {% for c in cohorts %} + + + + + + + + + + {% endfor %} + +
    NameDescriptionMembersAgentsCreated byCreatedActions
    + {{ c.name }} + + {% if c.description %}{{ c.description[:80] }}{% if c.description | length > 80 %}…{% endif %}{% else %}—{% endif %} + {{ c.memberships | length }} + {% for m in c.memberships | sort(attribute='agent_id') %}{{ gate.bot_names.get(m.agent_id, m.agent_id) }}{% if not loop.last %}, {% endif %}{% else %}—{% endfor %} + {{ creator_map.get(c.created_by | string, '—') }}{{ c.created_at.strftime('%b %d') }} + {% if c.memberships %} + {# Server refuses this too (v2 §12) — deleting a populated cohort + would cascade memberships away and silently reshape a running + simulation's topology. #} + Delete + {% else %} +
    + +
    + {% endif %} +
    + {% else %} +

    No cohorts yet. Create one above.

    + {% endif %} +
    +{% endblock %} diff --git a/templates/admin/discussions.html b/templates/admin/discussions.html index a00a08e..2fe3a41 100644 --- a/templates/admin/discussions.html +++ b/templates/admin/discussions.html @@ -123,7 +123,7 @@

    Discussions

    #{{ t.channel_name }} - {{ t.agent_id | capitalize }}Bot + {% if t.agent_id %}{{ t.agent_id | capitalize }}Bot{% else %}(unknown sender){% endif %} {{ t.reply_count }} {% if t.decision %} diff --git a/templates/agent/_thread_replies.html b/templates/agent/_thread_replies.html new file mode 100644 index 0000000..6c27338 --- /dev/null +++ b/templates/agent/_thread_replies.html @@ -0,0 +1,18 @@ +{# templates/agent/_thread_replies.html + Fragment returned by GET /agent/{agent_id}/thread/{message_ts}. Rendered + server-side so the gate and the markup stay in one place; the page injects it + verbatim. #} +{% if replies %} +
    + {% for m in replies %} +
    +
    + {{ m.sender }}{% if not m.is_bot %} · PI{% endif %} +
    +
    {{ m.content }}
    +
    + {% endfor %} +
    +{% else %} +

    No replies you can see in this thread.

    +{% endif %} diff --git a/templates/agent/conversations.html b/templates/agent/conversations.html new file mode 100644 index 0000000..101f6b8 --- /dev/null +++ b/templates/agent/conversations.html @@ -0,0 +1,153 @@ +{% extends "base.html" %} +{% block title %}Conversations — {{ agent.bot_name }} — CoPI{% endblock %} + +{% block content %} +
    +
    +
    +

    {{ agent.bot_name }} — Conversations

    +

    Post a message into your agent's workspace. Your agent picks it up on its next turn.

    +
    + ← Dashboard +
    + + {% if posted %} +
    + Message posted. Your agent will see it on its next turn. +
    + {% endif %} + + {% if not has_run %} +
    + No simulation run exists yet — there's nowhere to post. Once a run starts you can message your agent here. +
    + {% endif %} + + +
    +
    + + +
    + + +
    + +
    +
    + + +
    +

    Direct messages

    +

    Send a standing instruction ("always…", "never…") or a question. Your bot handles it like a Slack DM.

    + {% if dms %} +
    + {% for d in dms %} +
    + {{ d.content }} +
    + {% endfor %} +
    + {% endif %} +
    + + +
    +
    + + +

    Recent activity

    + {% if messages %} +
    + {% for m in messages %} +
    +
    + {{ m.sender }}{% if not m.is_bot %} · PI{% endif %} + #{{ m.channel }} +
    +
    {{ m.content }}
    + {% if m.reply_count and m.message_ts %} + + + {% endif %} +
    + {% endfor %} +
    + {% else %} +

    No messages yet in your agent's channels.

    + {% endif %} +
    + + +{% endblock %} diff --git a/templates/agent/dashboard.html b/templates/agent/dashboard.html index aebde37..512f884 100644 --- a/templates/agent/dashboard.html +++ b/templates/agent/dashboard.html @@ -292,6 +292,13 @@

    Reviewed Proposals

    View and edit your agent's private behavioral profile.

    + + +
    Conversations
    +

    See recent activity and post a message to your agent — no Slack required.

    +     + diff --git a/templates/base.html b/templates/base.html index 8e364d2..f664231 100644 --- a/templates/base.html +++ b/templates/base.html @@ -99,6 +99,7 @@ Activity Discussions Agents + Cohorts Access Waitlist diff --git a/templates/onboarding/add_texts.html b/templates/onboarding/add_texts.html deleted file mode 100644 index 2df72a1..0000000 --- a/templates/onboarding/add_texts.html +++ /dev/null @@ -1,76 +0,0 @@ -{% extends "base.html" %} -{% block title %}Add Supplementary Information — CoPI{% endblock %} - -{% block content %} -
    -
    -
    -

    Add Supplementary Information

    - Step 4 of 4 -
    -
    -
    -
    -
    - -
    -

    Supplementary Text Blocks

    -

    - You can add up to 5 text blocks — grant aims, equipment access, current research priorities, - or anything else that would help us generate a more accurate profile. These are - completely private and are never shown to other users or agents. -

    - - {% if profile and profile.user_submitted_texts %} -
    - {% for entry in profile.user_submitted_texts %} -
    -
    -

    {{ entry.label }}

    -

    {{ entry.content[:150] }}{% if entry.content | length > 150 %}...{% endif %}

    -
    - Remove -
    - {% endfor %} -
    - {% endif %} - - {% if not profile or not profile.user_submitted_texts or profile.user_submitted_texts | length < 5 %} -
    -
    - - -

    A short label to identify this text block.

    -
    -
    - - -

    Maximum 2,000 words.

    -
    - -
    - {% else %} -

    You've added the maximum of 5 text blocks.

    - {% endif %} -
    - -
    -
    - - -
    -
    -
    -{% endblock %} diff --git a/templates/onboarding/complete.html b/templates/onboarding/complete.html deleted file mode 100644 index e327a0f..0000000 --- a/templates/onboarding/complete.html +++ /dev/null @@ -1,36 +0,0 @@ -{% extends "base.html" %} -{% block title %}Welcome to CoPI{% endblock %} - -{% block content %} -
    -
    🎉
    -

    You're all set!

    -

    - Your research profile is ready. Other researchers at Scripps can now discover - collaboration opportunities with your lab. -

    - -
    -

    What's next?

    -
      -
    • - → - Review your profile — make sure everything is accurate and add any context we may have missed. -
      • -
    • - → - Edit your agent instructions — fine-tune your agent's private profile to control collaboration preferences, topic priorities, and communication style. -
      • -
    • - → - Your LabAgent is active — your AI agent in the Scripps research Slack workspace will begin exploring collaboration opportunities on your behalf. -
      • -
    -
    - - - View My Profile - -
    -{% endblock %} diff --git a/templates/profile/view.html b/templates/profile/view.html index 0bbc22d..e2e7d1c 100644 --- a/templates/profile/view.html +++ b/templates/profile/view.html @@ -38,23 +38,13 @@

    Welcome to CoPI!

    {% endif %} - {% if pending_profile %} - -
    -
    -
    -

    Updated Profile Available

    -

    - We found new publications and generated an updated profile. Review the changes below. -

    -
    - - Review Update - -
    -
    - {% endif %} + {# The "Updated Profile Available" banner that used to sit here linked to + GET /profile/review-update, a route b99fdfd never implemented — its only + control was a silent 404. Nothing in src/ writes + ResearcherProfile.pending_profile either, so the banner's condition never + held and there was no "changes below" for it to point at. Rebuild the + producer and the review page together if the feature is wanted; a banner + whose only action 404s is worse than no banner. #} {% if not profile %} diff --git a/tests/characterization/__snapshots__/test_profile_pipeline_gm.ambr b/tests/characterization/__snapshots__/test_profile_pipeline_gm.ambr index 0f0e750..fbc727e 100644 --- a/tests/characterization/__snapshots__/test_profile_pipeline_gm.ambr +++ b/tests/characterization/__snapshots__/test_profile_pipeline_gm.ambr @@ -13,6 +13,9 @@ 'disease_areas': list([ 'computational theory', ]), + 'evidence_pmid_count': 2, + 'evidence_pub_count': 2, + 'evidence_state': 'grounded', 'experimental_models': list([ 'analytical engine', 'difference engine', @@ -68,6 +71,7 @@ ]), 'raw_abstracts_hash': '66d887a29c4b61aa02e65852fb24d28cd464bede0c2950fbaf2f76ee13b9e42b', 'research_summary': 'This laboratory investigates programmable mechanical computation and the mathematical foundations that make general-purpose calculation possible. The central line of work formalizes how a sequence of operations can be encoded on punched cards and executed by an analytical engine, turning an abstract algorithm into a repeatable physical process. A recurring theme is the computation of Bernoulli numbers, used as a demanding proving ground for loop control, intermediate storage, and the reuse of partial results. The group connects nineteenth-century engine architecture to modern notions of symbolic manipulation, arguing that the machine could act on entities other than numbers when those entities obey formal rules. Methodologically the work spans analytical derivation, stepwise numerical verification, and the careful design of operation tables that other researchers can follow and reproduce.', + 'synthesis_validated': True, 'techniques': list([ 'mechanical computation', 'algorithm design', @@ -79,6 +83,9 @@ # name: test_profile_pipeline_llm_failure_leaves_fields_unset dict({ 'disease_areas': None, + 'evidence_pmid_count': None, + 'evidence_pub_count': None, + 'evidence_state': 'unknown', 'grant_titles': list([ 'Difference Engine Program', 'Analytical Engine Grant', @@ -87,11 +94,64 @@ 'profile_version': 0, 'raw_abstracts_hash_is_set': True, 'research_summary': None, + 'synthesis_validated': None, 'techniques': None, }) # --- +# name: test_profile_pipeline_marks_a_profile_that_fails_validation_twice + dict({ + 'disease_areas': list([ + ]), + 'evidence_pmid_count': 2, + 'evidence_pub_count': 2, + 'evidence_state': 'grounded', + 'llm_calls_total': 3, + 'profile_version': 1, + 'research_summary': 'The lab studies engines. Work continues on several fronts and results will be reported in due course elsewhere.', + 'synthesis_validated': False, + 'techniques': list([ + 'mechanical computation', + 'algorithm design', + ]), + 'unvalidated_in_progress': True, + }) +# --- +# name: test_profile_pipeline_orcid_works_failure_is_not_reported_as_no_works + dict({ + 'evidence_pmid_count': None, + 'evidence_pub_count': 0, + 'evidence_state': 'evidence_lost', + 'profile_version': 1, + }) +# --- +# name: test_profile_pipeline_pubmed_outage_on_rerun_keeps_the_grounded_profile + dict({ + 'evidence_pub_count': 2, + 'evidence_state': 'grounded', + 'first_version': 1, + 'generated_at_untouched': True, + 'rejected_in_progress': True, + 'second_version': 1, + }) +# --- +# name: test_profile_pipeline_pubmed_outage_stores_a_profile_marked_evidence_lost + dict({ + 'context_has_publications_section': False, + 'evidence_pmid_count': 2, + 'evidence_pub_count': 0, + 'evidence_state': 'evidence_lost', + 'profile_version': 1, + 'publication_rows': 0, + 'summary_is_set': True, + 'synthesis_validated': True, + 'ungrounded_in_progress': True, + 'validator_accepts_it': True, + }) +# --- # name: test_profile_pipeline_rerun_increments_version_and_updates_pubs dict({ + 'evidence_pmid_count': 2, + 'evidence_pub_count': 2, 'first_version': 1, 'llm_calls_total': 3, 'pub_count_after_two_runs': 2, @@ -99,5 +159,50 @@ 'second_version': 2, 'seed_set_after_first_run': True, 'seed_unchanged_on_rerun': True, + 'synthesis_validated': True, + }) +# --- +# name: test_profile_pipeline_rerun_that_fails_validation_keeps_the_stored_profile + dict({ + 'evidence_pub_count': 2, + 'first_version': 1, + 'kept_the_validated_summary': True, + 'llm_calls_total': 4, + 'profile_row_count': 1, + 'rejected_in_progress': True, + 'same_profile_row': True, + 'second_version': 1, + 'synthesis_validated': True, + 'took_the_rejected_draft': False, + }) +# --- +# name: test_profile_pipeline_researcher_with_no_works_is_not_reported_as_evidence_lost + dict({ + 'evidence_pmid_count': 0, + 'evidence_pub_count': 0, + 'evidence_state': 'no_evidence_available', + 'profile_version': 1, + 'summary_is_set': True, + 'synthesis_validated': True, + 'ungrounded_in_progress': True, + }) +# --- +# name: test_profile_pipeline_stores_the_retry_not_the_rejected_first_synthesis + dict({ + 'disease_areas': list([ + 'computational theory', + ]), + 'evidence_state': 'grounded', + 'llm_calls_total': 3, + 'profile_version': 1, + 'stored_the_rejected_draft': False, + 'stored_the_retry': True, + 'synthesis_validated': True, + 'techniques': list([ + 'mechanical computation', + 'algorithm design', + 'numerical analysis', + 'punch-card programming', + ]), }) # --- diff --git a/tests/characterization/test_auth_and_admin_routes.py b/tests/characterization/test_auth_and_admin_routes.py index 9a234c3..ffbd796 100644 --- a/tests/characterization/test_auth_and_admin_routes.py +++ b/tests/characterization/test_auth_and_admin_routes.py @@ -163,3 +163,104 @@ async def test_unsubscribe_invalid_token_renders_error_200(client): r = await client.get("/settings/unsubscribe/bogus-token") assert r.status_code == 200 assert "text/html" in r.headers["content-type"] + + +# --- /admin/discussions: nullable agent_id on Slack-imported posts ------------ + + +async def test_admin_discussions_survives_a_bot_post_with_no_agent_id(client, db_session): + """Regression: /admin/discussions 500'd in production. + + `_rebuild_state_from_slack` records real Slack messages whose sender cannot + be mapped to a known bot as `is_bot=True, agent_id=NULL` — measured: 7 such + rows, all from raw Slack user id U0BKJ6US485. The handler collected them + into `available_agents` unguarded (every sibling `.add()` IS guarded) and + then `sorted()` the set, so one NULL took the whole page down with + `TypeError: '<' not supported between instances of 'NoneType' and 'str'`. + """ + u = await factories.make_user(db_session, is_admin=True) + run = await factories.make_simulation_run(db_session) + # A normal bot post, so the set is genuinely mixed rather than all-None. + await factories.make_agent_message( + db_session, run=run, agent_id="gill", is_bot=True, + message_ts="1786000000.000100", thread_ts=None, channel_name="general", + ) + # The Slack-imported post with no mappable sender. + await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=True, + message_ts="1786000000.000200", thread_ts=None, channel_name="general", + sender_name="U0BKJ6US485", + ) + await db_session.flush() + + r = await client.get("/admin/discussions", headers=_auth_headers(u.id)) + assert r.status_code == 200 + # The NULL-agent thread's "Posted By" cell must not read as a real bot — + # `{{ t.agent_id | capitalize }}Bot` printed the literal "NoneBot". + assert "NoneBot" not in r.text + assert "(unknown sender)" in r.text + + +async def test_admin_activity_detail_survives_a_bot_post_with_no_agent_id(client, db_session): + """Regression: /admin/activity/{run_id} 500'd the same way /admin/discussions + did (fixed in 73a78c3 for that route only). + + `admin_activity_detail` builds `channel_stats[...]["agents"]` as a set and + does `channel_stats[channel]["agents"].add(msg.agent_id)` with no guard, then + `templates/admin/activity_detail.html` does `{{ stats.agents | sort | join(', ') }}` + — Jinja's `sort` is `sorted()`, so one NULL `agent_id` (the same + `_rebuild_state_from_slack` "sender maps to no known bot" case) takes the whole + page down with `TypeError: '<' not supported between instances of 'NoneType' + and 'str'`. This run is the one that is FIRST in the Activity table today, so + it is the most likely click in production. + """ + u = await factories.make_user(db_session, is_admin=True) + run = await factories.make_simulation_run(db_session) + # A normal bot post, so the set is genuinely mixed rather than all-None. + await factories.make_agent_message( + db_session, run=run, agent_id="gill", is_bot=True, + message_ts="1786000000.000100", thread_ts=None, channel_name="general", + ) + # The Slack-imported post with no mappable sender, in the same channel. + await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=True, + message_ts="1786000000.000200", thread_ts=None, channel_name="general", + sender_name="U0BKJ6US485", + ) + await db_session.flush() + + r = await client.get(f"/admin/activity/{run.id}", headers=_auth_headers(u.id)) + assert r.status_code == 200 + # The NULL-agent row must not be rendered as a lie: no literal "NoneBot" + # anywhere on the page (Messages-by-Agent table, by-channel agent list, or + # the message timeline). + assert "NoneBot" not in r.text + assert "(unknown sender)" in r.text + + +# --- /admin/activity/{run_id}/llm-calls: unbounded `page` --------------------- + + +async def test_admin_llm_calls_page_zero_rejected_not_500(client, db_session): + """Regression: `page: int = 1` had no lower bound, so `?page=0` computed + `offset = (0 - 1) * 50 = -50` and Postgres raised `OFFSET must not be + negative` as an unhandled 500. `Query(1, ge=1)` rejects an out-of-range + page with a clean 422 instead, and `?page=1` still works. + """ + u = await factories.make_user(db_session, is_admin=True) + run = await factories.make_simulation_run(db_session) + await db_session.flush() + + r = await client.get( + f"/admin/activity/{run.id}/llm-calls", + params={"page": 0}, + headers=_auth_headers(u.id), + ) + assert r.status_code == 422 + + r2 = await client.get( + f"/admin/activity/{run.id}/llm-calls", + params={"page": 1}, + headers=_auth_headers(u.id), + ) + assert r2.status_code == 200 diff --git a/tests/characterization/test_profile_pipeline_gm.py b/tests/characterization/test_profile_pipeline_gm.py index 0ddb94b..02962b3 100644 --- a/tests/characterization/test_profile_pipeline_gm.py +++ b/tests/characterization/test_profile_pipeline_gm.py @@ -18,7 +18,7 @@ import pytest from sqlalchemy import select -from src.models import Publication +from src.models import Job, Publication, ResearcherProfile from src.services import profile_pipeline from tests import factories from tests.fakes import FakeAnthropic @@ -55,6 +55,22 @@ "keywords": ["computing", "mathematics", "engines"], } +# A public-profile JSON that FAILS _validate_profile on all three of its rules: +# an 18-word summary (min 100), two techniques (min 3) and no disease areas. Every +# test that uses it re-asserts that it really is invalid, so the fixture cannot +# drift into validity and turn its test into a tautology. +_INVALID_PROFILE = { + "research_summary": ( + "The lab studies engines. Work continues on several fronts and results " + "will be reported in due course elsewhere." + ), + "techniques": ["mechanical computation", "algorithm design"], + "experimental_models": ["analytical engine"], + "disease_areas": [], + "key_targets": ["Bernoulli numbers"], + "keywords": ["computing"], +} + _PRIVATE_SEED = ( "# Private Profile\n\n" "## Collaboration Preferences\n" @@ -184,6 +200,13 @@ async def test_profile_pipeline_golden_master(db_session, monkeypatch, snapshot) "private_profile_md": profile.private_profile_md, "private_profile_seed": profile.private_profile_seed, "raw_abstracts_hash": profile.raw_abstracts_hash, + # Provenance of the synthesis (migration 0023). On the happy path the + # stored fields passed validation and both works carried an abstract, so + # the profile is grounded in the two publications below. + "synthesis_validated": profile.synthesis_validated, + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, "publications": pub_view, } @@ -195,7 +218,12 @@ async def test_profile_pipeline_golden_master(db_session, monkeypatch, snapshot) async def test_profile_pipeline_llm_failure_leaves_fields_unset(db_session, monkeypatch, snapshot): """Pin the resilience path: when the public-synthesis LLM call raises, the pipeline swallows it, stores no synthesized fields, and leaves version at 0 — - but still records grant titles and the abstracts hash and attempts the seed.""" + but still records grant titles and the abstracts hash and attempts the seed. + + The provenance columns stay NULL here, which is the third state they need: no + synthesis was stored, so there is nothing to say about its validation or its + evidence. `evidence_state` reads "unknown" rather than claiming the profile + had no evidence — it had no profile.""" _install_fakes(monkeypatch) # Replace the LLM with one that always raises on create(). @@ -222,6 +250,10 @@ def __init__(self): "profile_version": profile.profile_version, "private_profile_seed": profile.private_profile_seed, "raw_abstracts_hash_is_set": profile.raw_abstracts_hash is not None, + "synthesis_validated": profile.synthesis_validated, + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, } assert result == snapshot @@ -340,5 +372,428 @@ async def test_profile_pipeline_rerun_increments_version_and_updates_pubs( "seed_set_after_first_run": first_seed is not None, "seed_unchanged_on_rerun": second.private_profile_seed == first_seed, "llm_calls_total": len(fake_llm.calls), + # The provenance columns are rewritten each run, not accumulated: a second + # valid, grounded run over the same two publications leaves the same 2/2. + "synthesis_validated": second.synthesis_validated, + "evidence_pmid_count": second.evidence_pmid_count, + "evidence_pub_count": second.evidence_pub_count, + } + assert result == snapshot + + +# =========================================================================== +# Step 8/9: what the pipeline records about HOW a profile was produced. +# +# Until migration 0023 it recorded nothing, and two defects lived in that gap: +# +# * step 9 stored on `if synthesized:` alone, so the validation result — both +# the first one and the retry's — was computed and discarded. A profile that +# failed _validate_profile twice was persisted exactly like one that passed. +# * with PubMed unreachable, ORCID works never reach the prompt (they enter it +# only via their PubMed records), so the model invents a profile from a name +# and a department, it passes validation, profile_version is bumped, and zero +# Publication rows are written. +# +# Both were invisible to a black-box test because the outcome was byte-identical +# to the good path. The tests below are the ones that fail if _validate_profile +# is hardwired to `return True`, and the ones that tell a grounded profile from +# a fabricated one. +# =========================================================================== + + +def _progress_steps(job: Job) -> list[str]: + return [p["step"] for p in (job.payload or {}).get("progress", [])] + + +async def _make_job(db_session, user) -> Job: + """A real generate_profile Job, so update_progress writes where the worker and + the /onboarding page read it from (job.payload['progress']).""" + job = Job( + type="generate_profile", + user_id=user.id, + payload={"user_id": str(user.id), "orcid": user.orcid}, + ) + db_session.add(job) + await db_session.flush() + return job + + +async def test_profile_pipeline_stores_the_retry_not_the_rejected_first_synthesis( + db_session, monkeypatch, snapshot +): + """Validation fails on the first attempt and passes on the retry -> the RETRY + is what gets stored, and the profile is marked validated. + + This is the first of the three tests that die if `_validate_profile` is + hardwired to `return True`: with a validator that never says no, the retry + below never fires, the 18-word draft is stored instead of the good one, and + the LLM is called twice rather than three times. + """ + _install_fakes(monkeypatch) + assert profile_pipeline._validate_profile(_INVALID_PROFILE) is False, ( + "_INVALID_PROFILE now passes validation, so this test no longer exercises " + "the retry path it claims to" + ) + # public #1 (rejected) -> public #2 (accepted) -> private seed + fake_llm = FakeAnthropic( + [json.dumps(_INVALID_PROFILE), json.dumps(_VALID_PROFILE), _PRIVATE_SEED] + ) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + user = await factories.make_user( + db_session, name="Ada Lovelace", orcid="0000-0002-1825-0101", + ) + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session) + + result = { + "stored_the_retry": profile.research_summary == _VALID_PROFILE["research_summary"], + "stored_the_rejected_draft": ( + profile.research_summary == _INVALID_PROFILE["research_summary"] + ), + "techniques": profile.techniques, + "disease_areas": profile.disease_areas, + "profile_version": profile.profile_version, + "synthesis_validated": profile.synthesis_validated, + "evidence_state": profile.evidence_state, + "llm_calls_total": len(fake_llm.calls), + } + assert result == snapshot + # Cruxes, asserted explicitly so a careless --snapshot-update cannot bless a + # regression back to storing the rejected draft. + assert profile.research_summary == _VALID_PROFILE["research_summary"] + assert profile.synthesis_validated is True + assert len(fake_llm.calls) == 3, ( + f"{len(fake_llm.calls)} LLM calls; expected 3 (rejected public synthesis, " + "retry, private seed). 2 means the retry never fired, i.e. validation " + "accepted the invalid draft" + ) + + +async def test_profile_pipeline_marks_a_profile_that_fails_validation_twice( + db_session, monkeypatch, snapshot +): + """Validation fails BOTH times -> the draft is stored, and it is stored + *marked*: synthesis_validated=False, plus an 'unvalidated' entry in the job + progress the /onboarding page renders. + + Storing rather than discarding is the deliberate choice (see the step 9 + comment in profile_pipeline.py): the PI gets something to edit instead of an + unexplained empty form, and the mark is what makes the state distinguishable + and recoverable. What must never happen is what happened before 0023 — the + row looking exactly like a profile that passed. + + Second of the three mutation-killing tests: with `_validate_profile` hardwired + to `return True`, synthesis_validated comes out True, the progress entry is + absent, and only two LLM calls are made. + """ + _install_fakes(monkeypatch) + assert profile_pipeline._validate_profile(_INVALID_PROFILE) is False + # Both public attempts return the same invalid draft, then the private seed. + fake_llm = FakeAnthropic( + [json.dumps(_INVALID_PROFILE), json.dumps(_INVALID_PROFILE), _PRIVATE_SEED] + ) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + user = await factories.make_user( + db_session, name="Ada Lovelace", orcid="0000-0002-1825-0102", + ) + job = await _make_job(db_session, user) + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session, job=job) + + result = { + "research_summary": profile.research_summary, + "techniques": profile.techniques, + "disease_areas": profile.disease_areas, + # Still 1: the draft IS the stored profile, and the PI's onboarding page + # needs a profile to render. + "profile_version": profile.profile_version, + "synthesis_validated": profile.synthesis_validated, + # The draft is thin, but it is thin about real publications. + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, + "unvalidated_in_progress": "unvalidated" in _progress_steps(job), + "llm_calls_total": len(fake_llm.calls), + } + assert result == snapshot + # THE crux of fix 2. `is False`, not falsey: None means "nothing was ever + # synthesized here", which is a different state (see the LLM-failure GM). + assert profile.synthesis_validated is False, ( + "a profile that failed _validate_profile on both attempts was stored with " + f"synthesis_validated={profile.synthesis_validated!r}. If it is True the " + "validator is not being consulted; if it is None the store path did not " + "record the decision at all — either way step 9's gate is gone and a " + "below-standard profile is again indistinguishable from a good one" + ) + assert "unvalidated" in _progress_steps(job) + assert len(fake_llm.calls) == 3 + + +async def test_profile_pipeline_rerun_that_fails_validation_keeps_the_stored_profile( + db_session, monkeypatch, snapshot +): + """A monthly refresh whose synthesis fails validation must NOT overwrite the + good profile that is already stored. + + This is the case that makes "store the draft" safe: storing a marked draft is + right when there is nothing better, and wrong when there is. Before 0023 the + pipeline had no way to tell the difference, so the refresh clobbered. + + Third mutation-killing test: with `_validate_profile` hardwired to `return + True` the second run replaces the summary and bumps the version to 2. + """ + _install_fakes(monkeypatch) + assert profile_pipeline._validate_profile(_INVALID_PROFILE) is False + # Run 1: valid public synthesis + private seed. Run 2: invalid twice (the seed + # step is skipped because run 1 left a seed). + fake_llm = FakeAnthropic([ + json.dumps(_VALID_PROFILE), _PRIVATE_SEED, + json.dumps(_INVALID_PROFILE), json.dumps(_INVALID_PROFILE), + ]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + user = await factories.make_user( + db_session, name="Ada Lovelace", orcid="0000-0002-1825-0103", + ) + first = await profile_pipeline.run_profile_pipeline(user.id, db_session) + first_version = first.profile_version + job = await _make_job(db_session, user) + second = await profile_pipeline.run_profile_pipeline(user.id, db_session, job=job) + + rows = ( + await db_session.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == user.id) + ) + ).scalars().all() + + result = { + "first_version": first_version, + "second_version": second.profile_version, + "same_profile_row": first.id == second.id, + "profile_row_count": len(rows), + "kept_the_validated_summary": ( + second.research_summary == _VALID_PROFILE["research_summary"] + ), + "took_the_rejected_draft": ( + second.research_summary == _INVALID_PROFILE["research_summary"] + ), + "synthesis_validated": second.synthesis_validated, + "evidence_pub_count": second.evidence_pub_count, + "rejected_in_progress": "validation_rejected" in _progress_steps(job), + "llm_calls_total": len(fake_llm.calls), + } + assert result == snapshot + assert second.research_summary == _VALID_PROFILE["research_summary"], ( + "a synthesis that failed validation twice overwrote a profile that had " + "passed it — the monthly refresh now degrades profiles it cannot improve" + ) + assert second.profile_version == 1, ( + f"profile_version went to {second.profile_version} on a run that stored " + "nothing; the version must track the stored content, not the attempt" + ) + assert second.synthesis_validated is True + + +async def test_profile_pipeline_pubmed_outage_stores_a_profile_marked_evidence_lost( + db_session, monkeypatch, snapshot +): + """PubMed unreachable, ORCID and the LLM fine: the profile is fabricated from + a name and a department, and now says so. + + Every ingredient of the defect is reproduced: ORCID lists two works with + PMIDs, `fetch_pubmed_records` raises, so `pubs_for_synthesis` is empty, the + synthesis context contains no publication at all, ZERO Publication rows are + written — and the model still returns a profile that PASSES _validate_profile + (the fixture is the same valid one the happy path uses, which is exactly what + a real model does: it writes plausible prose from the name). + + Onboarding must still complete (asserted), so the discriminator cannot be a + refusal to store. It is the pair of evidence counts: 2 identifiers in hand, 0 + abstracts in the prompt -> evidence_lost. + """ + _install_fakes(monkeypatch) + + async def pubmed_is_down(pmids): + raise ConnectionError("simulated PubMed outage") + + monkeypatch.setattr(profile_pipeline, "fetch_pubmed_records", pubmed_is_down) + fake_llm = FakeAnthropic([json.dumps(_VALID_PROFILE), _PRIVATE_SEED]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + # Observe the prompt without replacing it: the claim "no publication reached + # the model" is about the real context builder's output. + contexts: list[str] = [] + real_ctx = profile_pipeline._build_synthesis_context + + def recording_ctx(**kwargs): + out = real_ctx(**kwargs) + contexts.append(out) + return out + + monkeypatch.setattr(profile_pipeline, "_build_synthesis_context", recording_ctx) + + user = await factories.make_user( + db_session, name="Ada Lovelace", orcid="0000-0002-1825-0104", + ) + job = await _make_job(db_session, user) + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session, job=job) + + pubs = ( + await db_session.execute(select(Publication).where(Publication.user_id == user.id)) + ).scalars().all() + + result = { + # Onboarding completed: there is a profile and it has a summary. + "profile_version": profile.profile_version, + "summary_is_set": bool(profile.research_summary), + # ...and it passed the shape validator, which is the whole problem: + # validation cannot see grounding. + "synthesis_validated": profile.synthesis_validated, + "validator_accepts_it": profile_pipeline._validate_profile( + { + "research_summary": profile.research_summary, + "techniques": profile.techniques, + "disease_areas": profile.disease_areas, + } + ), + # The discriminator. + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, + "publication_rows": len(pubs), + "context_has_publications_section": "## Publications" in contexts[0], + "ungrounded_in_progress": "ungrounded" in _progress_steps(job), } assert result == snapshot + # The crux: a fabricated profile is no longer indistinguishable from a real + # one. Both explicit, because either alone can be satisfied by accident — + # `evidence_pub_count == 0` also holds for a researcher with no papers, and + # only the PMID count separates "we lost it" from "there was none". + assert profile.evidence_pub_count == 0 and profile.evidence_pmid_count == 2 + assert profile.evidence_state == "evidence_lost" + assert len(pubs) == 0, ( + "Publication rows were written while PubMed was unreachable — they came " + "from somewhere other than PubMed and the count above means nothing" + ) + + +async def test_profile_pipeline_researcher_with_no_works_is_not_reported_as_evidence_lost( + db_session, monkeypatch, snapshot +): + """A genuinely publication-less researcher onboards, and is NOT confused with + an outage. + + Same observable surface as the test above — 0 abstracts in the prompt, 0 + Publication rows, a profile written from name and department — but nothing was + lost: ORCID was reachable and reported no works. An operator triaging + ungrounded profiles must not be sent to regenerate this one, because + regenerating cannot help. + """ + _install_fakes(monkeypatch) + + async def no_works(orcid_id): + return [] + + monkeypatch.setattr(profile_pipeline, "fetch_orcid_works", no_works) + fake_llm = FakeAnthropic([json.dumps(_VALID_PROFILE), _PRIVATE_SEED]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + user = await factories.make_user( + db_session, name="Josiah Carberry", orcid="0000-0002-1825-0105", + ) + job = await _make_job(db_session, user) + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session, job=job) + + result = { + "profile_version": profile.profile_version, + "summary_is_set": bool(profile.research_summary), + "synthesis_validated": profile.synthesis_validated, + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, + "ungrounded_in_progress": "ungrounded" in _progress_steps(job), + } + assert result == snapshot + assert profile.profile_version == 1, ( + "a researcher with no publications did not get a profile — a real, " + "publication-less PI must still be able to onboard" + ) + assert profile.evidence_state == "no_evidence_available", ( + f"reported {profile.evidence_state!r} for a researcher whose ORCID record " + "is simply empty; nothing was lost and regeneration cannot help, so this " + "must not be triaged as an outage" + ) + + +async def test_profile_pipeline_orcid_works_failure_is_not_reported_as_no_works( + db_session, monkeypatch, snapshot +): + """The inverse mistake: ORCID's works lookup FAILS, so the pipeline does not + know how many publications exist. Recording 0 identifiers would read as "this + researcher has no papers"; the count is left NULL and the state is + evidence_lost, which is the honest answer and the actionable one.""" + _install_fakes(monkeypatch) + + async def orcid_works_down(orcid_id): + raise ConnectionError("simulated ORCID outage") + + monkeypatch.setattr(profile_pipeline, "fetch_orcid_works", orcid_works_down) + fake_llm = FakeAnthropic([json.dumps(_VALID_PROFILE), _PRIVATE_SEED]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + user = await factories.make_user( + db_session, name="Ada Lovelace", orcid="0000-0002-1825-0106", + ) + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session) + + result = { + "profile_version": profile.profile_version, + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, + } + assert result == snapshot + assert profile.evidence_pmid_count is None and profile.evidence_state == "evidence_lost" + + +async def test_profile_pipeline_pubmed_outage_on_rerun_keeps_the_grounded_profile( + db_session, monkeypatch, snapshot +): + """The refresh case of the same defect: a monthly refresh that runs during a + PubMed outage must not replace a profile grounded in real abstracts with one + invented from a name. Both syntheses pass validation, so only the evidence + counts can tell the second one is worse — which is the reason they are + persisted rather than merely logged.""" + _install_fakes(monkeypatch) + fake_llm = FakeAnthropic( + [json.dumps(_VALID_PROFILE), _PRIVATE_SEED, json.dumps(_VALID_PROFILE)] + ) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake_llm) + + user = await factories.make_user( + db_session, name="Ada Lovelace", orcid="0000-0002-1825-0107", + ) + first = await profile_pipeline.run_profile_pipeline(user.id, db_session) + first_version = first.profile_version + first_generated_at = first.profile_generated_at + + async def pubmed_is_down(pmids): + raise ConnectionError("simulated PubMed outage") + + monkeypatch.setattr(profile_pipeline, "fetch_pubmed_records", pubmed_is_down) + job = await _make_job(db_session, user) + second = await profile_pipeline.run_profile_pipeline(user.id, db_session, job=job) + + result = { + "first_version": first_version, + "second_version": second.profile_version, + "evidence_pub_count": second.evidence_pub_count, + "evidence_state": second.evidence_state, + "generated_at_untouched": second.profile_generated_at == first_generated_at, + "rejected_in_progress": "validation_rejected" in _progress_steps(job), + } + assert result == snapshot + assert second.evidence_pub_count == 2 and second.profile_version == 1, ( + "a refresh during a PubMed outage replaced a profile grounded in 2 " + "abstracts with one grounded in none" + ) diff --git a/tests/characterization/test_public_routes.py b/tests/characterization/test_public_routes.py index 5c2ab6b..15a6983 100644 --- a/tests/characterization/test_public_routes.py +++ b/tests/characterization/test_public_routes.py @@ -162,3 +162,37 @@ async def test_proposal_vote_details_happy_path_ok(client, db_session): ) assert r.status_code == 200 assert r.json() == {"ok": True} + + +@pytest.mark.parametrize("bad_payload", [{}, {"voter_token": "not-the-owner"}]) +async def test_proposal_vote_details_wrong_or_absent_token_403( + client, db_session, bad_payload +): + """SEC: `if vote_obj.voter_token and token and vote_obj.voter_token != token` + short-circuited to False whenever the caller simply omitted `voter_token` + from the body (token is None) — so anyone holding a vote_id could overwrite, + or erase (`details: null`), another visitor's free-text comment with no + token at all. A wrong token, and an altogether absent one, must both be + rejected whenever the stored row actually has a token.""" + d = await factories.make_thread_decision( + db_session, outcome="proposal", origin_visibility="public" + ) + created = await client.post( + "/api/proposal-vote", + json={"decision_id": str(d.id), "vote": "up", "voter_token": "browser-tok-4"}, + ) + vote_id = created.json()["id"] + r = await client.post( + f"/api/proposal-vote/{vote_id}/details", + json={"details": "attacker-supplied", **bad_payload}, + ) + assert r.status_code == 403 + + # The original comment/attempted tamper must not have landed: the correct + # token still works and is the only way to update this row. + r2 = await client.post( + f"/api/proposal-vote/{vote_id}/details", + json={"details": "legit update", "voter_token": "browser-tok-4"}, + ) + assert r2.status_code == 200 + assert r2.json() == {"ok": True} diff --git a/tests/conftest.py b/tests/conftest.py index b2ecd74..3b5245c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,13 +27,23 @@ @pytest.fixture(scope="session") def _pg_container(): + # Allow pointing the suite at an already-running Postgres via TEST_DATABASE_URL + # (an asyncpg DSN for a throwaway DB). This avoids requiring a Docker socket for + # testcontainers — e.g. when running inside the app container, which has none. + # Default behavior (ephemeral container) is unchanged when the var is unset. + if os.environ.get("TEST_DATABASE_URL"): + yield None + return with PostgresContainer("postgres:15", dbname="copi_test") as pg: yield pg @pytest.fixture(scope="session") def pg_url(_pg_container): - """asyncpg DSN for the ephemeral container (creds come from the container, not hardcoded).""" + """asyncpg DSN for the test DB (creds come from the container/env, not hardcoded).""" + env_url = os.environ.get("TEST_DATABASE_URL") + if env_url: + return env_url return ( f"postgresql+asyncpg://{_pg_container.username}:{_pg_container.password}" f"@{_pg_container.get_container_host_ip()}:{_pg_container.get_exposed_port(5432)}" @@ -118,3 +128,217 @@ async def _override_get_db(): @pytest.fixture def _text(): return text + + +# --------------------------------------------------------------------------- +# Live Slack tier — see .notes/slack-integration-test-plan.md +# --------------------------------------------------------------------------- + +_LIVE_SLACK_ENV = ("SLACK_TEST_WORKSPACE", "SLACK_TEST_PI_USER_ID", + "SLACK_TEST_BOT_TOKEN_SU") + + +def pytest_collection_modifyitems(config, items): + """Skip the live Slack tier unless the workspace credentials are present. + + Deliberately a skip rather than a collection filter, so `-m live_slack` with no + credentials reports "skipped" instead of "no tests ran" — the latter is + indistinguishable from a typo'd marker. + """ + missing = [k for k in _LIVE_SLACK_ENV if not os.environ.get(k)] + if missing: + skip = pytest.mark.skip(reason=f"live Slack tier needs {', '.join(missing)}") + for item in items: + if "live_slack" in item.keywords: + item.add_marker(skip) + + if not os.environ.get("LIVE_API_TESTS"): + skip_api = pytest.mark.skip(reason="live third-party API tier needs LIVE_API_TESTS=1") + for item in items: + if "live_api" in item.keywords: + item.add_marker(skip_api) + + +@pytest.fixture(scope="session") +def slack_bot_tokens() -> dict[str, str]: + """Bot tokens from the environment, keyed by agent_id. Never read from a file.""" + out = {} + for aid in ("su", "cravatt", "wiseman"): + tok = os.environ.get(f"SLACK_TEST_BOT_TOKEN_{aid.upper()}", "") + if tok: + out[aid] = tok + return out + + +@pytest.fixture(scope="session") +def slack_pi_user_id() -> str: + return os.environ.get("SLACK_TEST_PI_USER_ID", "") + + +def _make_slack_client(agent_id: str, token: str, visibility_lookup=None): + from src.agent.slack_client import AgentSlackClient + + c = AgentSlackClient(agent_id=agent_id, bot_token=token, + visibility_lookup=visibility_lookup) + assert c.connect() is True, f"[{agent_id}] auth.test failed — token dead or revoked" + return c + + +_PROBE_BOTS = ("su", "cravatt", "wiseman") + + +@pytest.fixture +def slack_bot_tokens_all(slack_bot_tokens): + """All three probe tokens, or skip — the raw-token analogue of ``slack_clients``. + + A test that compares the bots *against each other* (same workspace? distinct bot + users? which scopes were granted?) needs every token, and there is no partial + answer: with one token those comparisons are vacuous, so passing would be worse + than skipping. + + This exists because two tests in test_slack_provision_live.py took + ``slack_bot_tokens`` directly and then indexed ``["wiseman"]``, so an environment + holding only su's token FAILED them — reporting an incomplete environment as a + product defect. ``pytest_collection_modifyitems`` gates the whole tier on + SLACK_TEST_BOT_TOKEN_SU alone, so su-only is an expected, supported state and the + tier's tests have to say what more they need. + """ + missing = [a for a in _PROBE_BOTS if a not in slack_bot_tokens] + if missing: + pytest.skip(f"no bot token for {missing}") + return slack_bot_tokens + + +@pytest.fixture +def slack_clients(slack_bot_tokens): + """All three probe clients, connected. Skips if any token is absent.""" + missing = [a for a in _PROBE_BOTS if a not in slack_bot_tokens] + if missing: + pytest.skip(f"no bot token for {missing}") + return {a: _make_slack_client(a, t) for a, t in slack_bot_tokens.items()} + + +@pytest.fixture +def slack_client_su(slack_clients): + return slack_clients["su"] + + +@pytest.fixture +def slack_list_all_channels(): + """Fully paginated conversations.list — the ground truth for "does Slack have this + channel". Returns a callable ``(client, include_private=False) -> {name: id}``. + + Originally needed because ``AgentSlackClient.list_channels`` asked for a single + 200-item page and ignored ``response_metadata.next_cursor``, so on a workspace with + more than 200 conversations it returned an arbitrary *subset*: Slack orders + conversations.list by channel id, ids are not monotonic in creation time, and this + workspace has 300+ public channels (most of them archived `t-` channels from earlier + runs — Slack has no delete-channel API). Asking ``list_channels()`` whether a channel + existed was therefore a coin flip, and that was the cause of the whole live tier's + rotating failures. + + ``list_channels`` paginates now, so this fixture is no longer a *workaround*. It is + kept because it is a deliberately **independent** implementation: a test that asked + the client's own listing whether the client's own listing was complete would pass + just as happily if both shared a bug. Assertions about the client's completeness are + made against this, in + ``test_slack_client_live.py::test_list_channels_returns_every_public_channel``, which + no longer carries an xfail marker. + + One property to respect at the call site: this is a cursor walk, not a snapshot. The + suite mutates the workspace as it runs (probe channels are created and archived by + fixtures) and Slack's listing is eventually consistent, so a channel can be absent + from one complete walk and present in the next seconds later. Tests that compare two + walks bracket the call under test and assert against both — see that test for the + measurement. + """ + def _all(client, *, include_private: bool = False) -> dict[str, str]: + types = "public_channel,private_channel" if include_private else "public_channel" + out: dict[str, str] = {} + cursor = "" + while True: + r = client._call_with_retry( + client._client.conversations_list, + types=types, limit=200, cursor=cursor, + ) + for ch in r.get("channels", []): + out[ch["name"]] = ch["id"] + cursor = (r.get("response_metadata") or {}).get("next_cursor") or "" + if not cursor: + return out + + return _all + + +@pytest.fixture +def slack_probe_channel(slack_clients): + """A fresh `t-`-prefixed public channel, archived on teardown. + + Slack has no delete-channel API, so this archives. The `t-` prefix means a test can + never touch one of the seeded channel names in src/agent/channels.py, and the + teardown script can match on it safely. + + The name->id cache of *every* client is seeded, not just the creator's. This mirrors + what the engine does in production — `_ensure_seeded_channels` ends with + `for c in self.slack_clients.values(): c.cache_channel_ids(existing)` + (src/agent/simulation.py:3061) — and it is load-bearing here rather than cosmetic: + the engine's `_post_message` passes a channel *name* to `post_message`, which + resolves it through `_resolve_channel_id` -> `list_channels()`. Only the creating + client gets a cache entry from `create_channel`, so a post by any other agent used to + depend on whether this brand-new channel happened to land in Slack's first 200-item + page — see the slack_list_all_channels docstring. When it did not, the name was + passed through to chat.postMessage verbatim and Slack answered `not_in_channel`, at + random, in whichever tests happened to post as cravatt or wiseman. + """ + import uuid as _uuid + + su = slack_clients["su"] + name = f"t-probe-{_uuid.uuid4().hex[:8]}" + data = su.create_channel(name) + assert data and data.get("id"), f"could not create #{name}: {data}" + for c in slack_clients.values(): + c.cache_channel_ids({name: data["id"]}) + yield name, data["id"] + try: + su._call_with_retry(su._client.conversations_archive, channel=data["id"]) + except Exception as exc: # teardown must not mask a test failure + print(f"WARNING: could not archive #{name}: {exc}") + + +@pytest.fixture(scope="session") +def api_budget(): + """Per-provider rate limiting and a call ceiling for the live_api tier. + + NCBI blocks clients that exceed 3 req/s without a key and *requires* tool= and + email= on every request. ORCID and grants.gov are more forgiving but a runaway loop + can still get the IP throttled, which would break the tier for everyone afterwards. + """ + import time as _time + + limits = {"ncbi": 0.40, "orcid": 0.10, "grants": 1.0} + last: dict[str, float] = {} + counts: dict[str, int] = {} + + class Budget: + max_calls = 200 + + def wait(self, provider: str): + gap = limits.get(provider, 0.5) + prev = last.get(provider) + if prev is not None: + delta = _time.monotonic() - prev + if delta < gap: + _time.sleep(gap - delta) + last[provider] = _time.monotonic() + counts[provider] = counts.get(provider, 0) + 1 + total = sum(counts.values()) + assert total <= self.max_calls, ( + f"live_api call ceiling exceeded ({total} > {self.max_calls}); " + f"per-provider: {counts}. A test is looping." + ) + + @property + def counts(self): + return dict(counts) + + return Budget() diff --git a/tests/contract/test_grants_contract.py b/tests/contract/test_grants_contract.py index e466799..4c7d658 100644 --- a/tests/contract/test_grants_contract.py +++ b/tests/contract/test_grants_contract.py @@ -34,7 +34,6 @@ async def test_search_opportunities_maps_fields(): "agencyCode": "HHS-NIH11", "openDate": "2026-01-01", "closeDate": "2026-06-01", - "description": "Study immunology.", } respx.post(SEARCH_URL).mock(return_value=httpx.Response(200, json=_search_payload([hit]))) results = await grants.search_opportunities("immunology") @@ -46,7 +45,11 @@ async def test_search_opportunities_maps_fields(): "agency": "HHS-NIH11", "open_date": "2026-01-01", "close_date": "2026-06-01", - "description": "Study immunology.", + # The provider literal above sends no description — search2 does not, + # measured live 2026-08-04 — so the projection maps it to "". Asserting + # the empty string rather than dropping the key keeps both halves + # honest: what grants.gov sends, and what we hand our callers. + "description": "", } ] @@ -131,7 +134,7 @@ async def test_fetch_opportunity_detail_synopsis_non_dict_is_blank(): @respx.mock async def test_search_for_researchers_dedups_by_number_and_tags_keyword(): hit = {"id": 1, "number": "N1", "title": "T1", "agencyCode": "NSF", - "openDate": "", "closeDate": "", "description": ""} + "openDate": "", "closeDate": ""} route = respx.post(SEARCH_URL).mock(return_value=httpx.Response(200, json=_search_payload([hit]))) out = await grants.search_for_researchers({"agent1": ["kw-a", "kw-b"]}) # Both keywords were actually searched (not short-circuited) — without this, a bug diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..348fa21 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,235 @@ +# `tests/e2e` — browser flows (Task 12) + +Covers `.notes/full-system-test-plan.md` §"Task 12". Two things live here: + +- **`test_browser_flows.py`** — `FLOWS`, a machine-readable transcript of each + flow (what to open, what to click, what must be visible), plus HTTP replays of + every flow whose steps are ordinary form posts. +- **`seed.py` / `session.py` / `mint_cookie.py` / `auth_helper.py`** — the + harness. `seed.py` writes fixture rows to a **live** database; the other three + exist because ORCID login is broken (below). + +Default `pytest tests/` behaviour: only the two offline well-formedness tests +run; the rest skip for want of `E2E_BASE_URL`. Nothing here touches the +production `copi` database — `seed.py` refuses any database not on +`ALLOWED_DATABASES`. + +## READ THIS FIRST — `copi_slack_test` must never be dropped + +Running the Slack provisioning flow rotates the Slack **app-configuration** +credential pair. Rotation is single-use: the token you type in is dead +afterwards, and the replacement `(slack_config_token, slack_config_refresh_token, +slack_config_token_exp)` triple is written into `app_settings` of whichever +database the request used. That is `copi_slack_test`. **Drop that database and +Slack app-configuration access is gone permanently**, with no way to recover it. +See `.notes/slack-integration-test-plan.md` §"Global Constraints". + +## Setup + +```bash +# 0. Migrate the e2e database first, and re-check this EVERY time. It is not +# permanently at head: `copi_slack_test` was left at 0022 while the branch head +# moved to 0023, and because the ORM carries 0023's columns +# (ResearcherProfile.synthesis_validated and the two evidence counts) any query +# that touches researcher_profiles then fails with UndefinedColumnError — which +# takes out `python -m tests.e2e.seed` and the whole onboarding flow. Adding +# three nullable columns is safe; NEVER downgrade this database (see above). +docker compose exec -T \ + -e DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_slack_test \ + app python -m alembic upgrade head + +# 1. an app instance on a MIGRATED database (the live `copi` DB is at 0018 and +# has no `agents` table, so /admin/agents cannot work against it) +docker compose run -d --name app-8002 -p 8002:8000 \ + -e DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_slack_test \ + -e BASE_URL=http://localhost:8002 -e ALLOW_HTTP_SESSIONS=true \ + app uvicorn src.main:app --host 0.0.0.0 --port 8000 + +# 2. a SECOND instance on the SAME database with isolation on. Cohort settings +# are read once per process, so the banner control needs a second process. +docker compose run -d --name app-8003 -p 8003:8000 \ + -e DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_slack_test \ + -e BASE_URL=http://localhost:8003 -e ALLOW_HTTP_SESSIONS=true \ + -e COHORT_ISOLATION_ENABLED=true -e COHORT_DEFAULT_POLICY=isolated \ + app uvicorn src.main:app --host 0.0.0.0 --port 8000 + +# 3. seed, and note the printed user ids +docker exec -i app-8002 python -m tests.e2e.seed + +# 4. run. Container-to-container hostnames, because pytest runs inside `app`. +docker compose exec -T \ + -e E2E_BASE_URL=http://app-8002:8000 \ + -e E2E_ISOLATION_BASE_URL=http://app-8003:8000 \ + -e E2E_ADMIN_USER_ID= \ + -e E2E_SIGNUP_USER_ID= \ + -e E2E_ONBOARDING_USER_ID= \ + app python -m pytest tests/e2e/test_browser_flows.py -q +``` + +Steps 1 and 2 are `docker compose run`, which *creates* a container, so on any +machine that has run this before they fail with `Conflict. The container name +"/app-8002" is already in use`. The two containers carry their env and port +bindings, and `.:/app` is a mount, so the right move is to reuse them — +`docker start app-8002 app-8003` — and, since neither runs `--reload`, +`docker restart app-8002 app-8003` after any `src/` change you want under test. + +Or run step 4 from the **host** against `.venv-test` — the same interpreter +`scripts/ci.sh` uses, so a green tier here is green under the gate. Both app +instances publish host ports, so the URLs are `localhost:` rather than the +container hostnames; everything else is identical, and the forged cookie works +because `src.config` reads the same `.env` `SECRET_KEY` the containers do: + +```bash +E2E_BASE_URL=http://localhost:8002 \ +E2E_ISOLATION_BASE_URL=http://localhost:8003 \ +E2E_ADMIN_USER_ID= \ +E2E_SIGNUP_USER_ID= \ +E2E_ONBOARDING_USER_ID= \ +.venv-test/bin/python -m pytest tests/e2e/test_browser_flows.py -q +# -> 8 passed, 1 xfailed (the xfail is the ORCID pin below) +``` + +Nothing in that command needs a browser: the pytest tier is httpx replays. The +**driven** half — replaying `FLOWS` in a real browser for the screenshots in +`.playwright-mcp/` — is what needs Playwright, and the wheel alone is not enough: + +```bash +uv pip install --python .venv-test/bin/python playwright # or: -e '.[dev]' +.venv-test/bin/python -m playwright install chromium # ~300MB of binaries +``` + +## Authentication: why the cookie is forged + +`session.py` forges the signed `copi-session` cookie exactly as +`tests/integration/test_cohort_admin.py::_auth` does — +`itsdangerous.TimestampSigner(settings.secret_key)` over +`base64(json({"user_id": ...}))`. + +This is not a convenience. **ORCID login does not work in this deployment**: +`.env` carries `ORCID_CLIENT_ID=test-client-id`, and ORCID's authorize endpoint +answers + +``` +HTTP 400 {"error":"invalid_request","error_description":"Invalid parameter: client_id"} +``` + +so there is no consent screen for any browser to click. Worse, ORCID degrades +that 400 into its ordinary sign-in page, so the user sees a plausible ORCID +screen that can never redirect back to `/auth/callback`, and our side logs +nothing at all. `test_orcid_login_cannot_be_driven_and_says_why` xfails on this +so the finding cannot be lost. The fix is configuration (real ORCID OAuth +credentials), not code. + +Two gotchas that cost red tests and are pinned in comments: + +- Put the forged cookie in the **cookie jar**, never in a per-request `Cookie` + header. The app re-issues `copi-session` on every response, and httpx then + sends the jar cookie *plus* the explicit header; Starlette joins the two + header values with `", "`, fails to parse, and the request arrives + **unauthenticated**. It only bites on a redirect-follow, i.e. exactly on the + form posts. +- Never share an `httpx.Client` across tests — it carries the previous + identity's cookie. + +## What needs a human, and why + +| flow | automatable? | why | +|---|---|---| +| admin: create cohort + edit topology | yes | ordinary form posts | +| agent self-service signup | yes | ordinary form post | +| public graph | yes | unauthenticated GET | +| onboarding | partly — see below | | +| **Slack provisioning** | **no** | needs a Slack-authenticated browser | +| **ORCID login** | **no** | no valid client_id; no consent screen exists | + +### Slack provisioning + +Slack's OAuth consent ("Allow") screen requires a browser with a live Slack +session, and Slack offers no headless install grant. The Playwright/MCP browser +has no Slack session, so this one step is irreducibly human. Everything either +side of it is real code under test: the `Provision` POST calls +`apps.manifest.create` for real, and Slack's redirect lands on the app's own +`/admin/agents/slack/callback`, which runs `complete_provisioning`. + +Procedure: + +1. Arm the app process with the config **refresh** token (env var only — never a + file, never a command-line argument): + re-create `app-8002` with `-e SLACK_CONFIG_REFRESH_TOKEN=...`. +2. Serve the admin cookie to the human's browser: + + ```bash + docker exec -i app-8002 python -m tests.e2e.mint_cookie + E2E_SESSION_COOKIE='' \ + E2E_TARGET_URL='http://localhost:8002/admin/agents/' \ + E2E_HELPER_PORT=8099 python3 tests/e2e/auth_helper.py + ``` + + Cookies ignore port, so a cookie set by `localhost:8099` with no `Domain` + attribute is sent to `localhost:8002` too. Caveat for the human: it is + host-scoped to `localhost`, so it replaces their session on every localhost + port. +3. Human opens `http://localhost:8099/`, clicks **Provision**, **verifies the + workspace name is the test workspace**, clicks **Allow**, and lands on + `/admin/agents/?slack_ok=1`. +4. Verify from **Slack's** side, not from our column: `auth.test` on the + resulting `xoxb-` token must return `ok: true` with the test workspace's + `team_id` and a `bot_id`. A set `slack_bot_token` column proves only that we + wrote a column. + +The agent's `bot_name` **must end in `ProbeBot`** — +`scripts/slack_test_teardown.py` deletes apps by that suffix and refuses to +touch anything else, so a differently-named bot becomes permanent workspace +litter. + +> **Record the `app_id` yourself.** `exchange_code` returns only the `xoxb-` +> string and drops the `app_id` from the `oauth.v2.access` response, and +> `complete_provisioning` then deletes the `SlackAppProvision` row that held it. +> Nothing in the data model keeps it. Since config tokens have no list-apps API, +> an app provisioned through the admin UI cannot afterwards be found for deletion +> or audit; `slack_test_teardown.py` enumerates from `PROBE_APPS_JSON` for +> exactly that reason. Recover it with one read-only call — +> `bots.info?bot=` returns `app_id` — and write it down. +> +> Provisioned by this task on 2026-07-31: `T12ProbeBot`, `app_id=A0BM5AY6HEW`, +> `bot_user=U0BMZLWQARE`, `bot_id=B0BM78V63UH`, workspace `copi-test` +> (`T0BMVSBMEC8`). + +`status='pending'` → `'active'` (the **Approve & Activate** button) is a +*separate* step and is **not** covered here: provisioning writes the token and +leaves the status alone. Covering it needs one more POST to +`/admin/agents/{id}/approve` plus an assertion that a running `agent-run` picks +the agent up on its next `_sync_roster_from_db` (~30s) — which needs an engine +process, so it belongs with Task 13, not here. + +### Onboarding + +The onboarding *routes* are fully drivable with a session: start → review → +private profile → complete, with `users.onboarding_complete` flipping only on +the final POST. + +It stops on its own at **Step 3 of 4, "Building Your Profile"**. `/onboarding` +auto-enqueues a `generate_profile` job and shows that spinner while +`job_status` is `none`/`pending`/`processing`. Advancing needs the worker to run +`run_profile_pipeline`, which fetches the user's ORCID record — so without +usable ORCID credentials it can never complete, and the page spins forever. The +flow therefore substitutes the `ResearcherProfile` row the pipeline would have +written and continues from there; the pipeline itself is Task 4's subject. + +**The flow destroys its own fixture, so `seed.py` resets it.** Walking it sets +`users.onboarding_complete=True`, and the substitute step leaves a +`ResearcherProfile` plus a `generate_profile` job in status `completed`. Any of +the three and a re-run is not the same test: `onboarding_complete` makes +`/onboarding` 302 straight to `/profile`, and a `completed` job takes +`profile_review.html` past the spinner branch. Get-or-create does not undo that — +it finds the row and leaves it — so `seed.py` now explicitly clears all three for +`ONBOARDING_ORCID` and nothing else. Found on 2026-08-04: all three were still +set from the 2026-07-31 run, and +`test_onboarding_goes_as_far_as_the_orcid_dependency` had been silently falling +through to its second branch. **Re-seed before trusting that test**; if it reports +the completed-profile branch, the reset did not run. + +## Artefacts + +Screenshots and accessibility snapshots from the driven runs land in +`.playwright-mcp/` (gitignored), prefixed `t12-`. diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..124eaa9 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1 @@ +"""Browser-driven end-to-end flows (Task 12 of .notes/full-system-test-plan.md).""" diff --git a/tests/e2e/auth_helper.py b/tests/e2e/auth_helper.py new file mode 100644 index 0000000..90fe608 --- /dev/null +++ b/tests/e2e/auth_helper.py @@ -0,0 +1,84 @@ +"""One-shot cookie-planting redirector, so a *human's* browser can be logged in. + +Why this exists +--------------- +Two of the Task 12 flows cannot be driven by the automation browser: + +* **Slack OAuth approval.** The Playwright browser has no Slack session, so + Slack's "Allow" screen cannot be reached, let alone clicked. +* Anything downstream of it, because Slack redirects to + ``/admin/agents/slack/callback`` which is behind ``get_admin_user``. + +So the human drives it in *their* browser, which is signed into Slack. They +still need our admin session cookie, and ORCID login is broken (see +``tests/e2e/README.md``). This server hands it to them: it sets the +pre-signed ``copi-session`` cookie for host ``localhost`` and 302s to the app. + +**Cookies ignore port.** A cookie set by ``localhost:8099`` with no ``Domain`` +attribute is host-only for ``localhost`` and is therefore sent to +``localhost:8002`` as well. That is the whole trick — this process never needs +to be the app. + +Usage (host, stdlib only — no venv needed):: + + # 1. mint the cookie inside a container that has the app's SECRET_KEY + docker exec -i app-8002 python -m tests.e2e.mint_cookie + + # 2. serve it + E2E_SESSION_COOKIE='' \ + E2E_TARGET_URL='http://localhost:8002/admin/agents' \ + python3 tests/e2e/auth_helper.py + + # 3. give the human http://localhost:8099/ + +Caveat to tell the human: the cookie is scoped to ``localhost``, so it replaces +any session they had on *any* localhost port, including the 8001 instance. +""" + +import http.server +import os +import sys + +PORT = int(os.environ.get("E2E_HELPER_PORT", "8099")) +COOKIE_NAME = "copi-session" + + +class _Handler(http.server.BaseHTTPRequestHandler): + cookie = "" + target = "" + + def do_GET(self): # noqa: N802 - BaseHTTPRequestHandler API + if self.path.rstrip("/") not in ("", "/go"): + self.send_error(404) + return + self.send_response(302) + # No Domain attribute => host-only cookie for "localhost", which the + # browser sends to every port on that host. + self.send_header( + "Set-Cookie", + f"{COOKIE_NAME}={self.cookie}; Path=/; SameSite=Lax; Max-Age=86400", + ) + self.send_header("Location", self.target) + self.send_header("Cache-Control", "no-store") + self.end_headers() + + def log_message(self, fmt, *args): + sys.stderr.write(f"[auth_helper] {fmt % args}\n") + + +def main() -> None: + cookie = os.environ.get("E2E_SESSION_COOKIE", "") + target = os.environ.get("E2E_TARGET_URL", "") + if not cookie or not target: + sys.exit("E2E_SESSION_COOKIE and E2E_TARGET_URL are required") + _Handler.cookie = cookie + _Handler.target = target + httpd = http.server.HTTPServer(("127.0.0.1", PORT), _Handler) + sys.stderr.write( + f"[auth_helper] http://localhost:{PORT}/ -> sets {COOKIE_NAME} -> {target}\n" + ) + httpd.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/mint_cookie.py b/tests/e2e/mint_cookie.py new file mode 100644 index 0000000..b599ebe --- /dev/null +++ b/tests/e2e/mint_cookie.py @@ -0,0 +1,27 @@ +"""Print a signed ``copi-session`` cookie value for a user id. + +Must run where the app's ``SECRET_KEY`` is readable (i.e. inside a container), +because the cookie is only accepted by ``SessionMiddleware`` if it is signed +with that key:: + + docker exec -i app-8002 python -m tests.e2e.mint_cookie + +This is the same forgery ``tests/integration/test_cohort_admin.py::_auth`` does, +extracted so the host-side helper can consume it. It is a *test* affordance for +a broken login path, not a production one: it needs the signing key, so it +grants nothing an operator does not already have. +""" + +import sys + +from tests.e2e.session import forge_session_cookie + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: python -m tests.e2e.mint_cookie ") + print(forge_session_cookie(sys.argv[1])) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/seed.py b/tests/e2e/seed.py new file mode 100644 index 0000000..1d20df3 --- /dev/null +++ b/tests/e2e/seed.py @@ -0,0 +1,326 @@ +"""Seed the browser-flow (e2e) database. + +Run *inside* a container whose ``DATABASE_URL`` points at the e2e database +(``copi_slack_test``), which is already at alembic head:: + + docker exec -i app-8002 python -m tests.e2e.seed + +Idempotent: every row is looked up by its natural key first, so re-running only +tops up what is missing. It commits for real — unlike the rest of the suite this +module writes to a live database on purpose, because the flows it supports are +driven by a browser against a running server, not by an ASGI transport inside a +rolled-back transaction. + +**Never point this at the production ``copi`` database.** It refuses any +database name that is not on ``ALLOWED_DATABASES``. + +What it creates, and which flow needs it: + +=========================== ==================================================== +row used by +=========================== ==================================================== +``ADMIN_ORCID`` user every ``/admin/**`` flow (forged session cookie) +``SIGNUP_ORCID`` user agent self-service signup (``POST /agent/request``) +``ONBOARDING_ORCID`` user the onboarding walk (deliberately no profile/job) +``PROBE_AGENT_ID`` agent Slack provisioning (``*ProbeBot``, status=pending) +5 Scripps agents + edges ``/scripps-graph`` and ``/cabo-graph`` render +=========================== ==================================================== +""" + +import asyncio +import os +import sys +from datetime import UTC, datetime + +from sqlalchemy import delete, select + +# Identities the browser flows log in as. ORCIDs are in the ISNI test range that +# orcid.org never issues, so these rows can never collide with a real login. +ADMIN_ORCID = "0000-0002-0000-9001" +ADMIN_EMAIL = "e2e-admin@example.org" +SIGNUP_ORCID = "0000-0002-0000-9002" +SIGNUP_EMAIL = "e2e-signup@example.org" +ONBOARDING_ORCID = "0000-0002-0000-9003" +ONBOARDING_EMAIL = "e2e-onboarding@example.org" + +# The agent provisioned against real Slack. The name MUST end in "ProbeBot": +# scripts/slack_test_teardown.py deletes apps by that suffix and refuses to +# touch anything else, so a bot named otherwise becomes permanent litter in the +# workspace. +PROBE_AGENT_ID = "t12probe" +PROBE_BOT_NAME = "T12ProbeBot" + +# Graph fixture. agent_ids are drawn from src/routers/public.py::_SCRIPPS so the +# scripps_only node filter keeps them; the edge set is a connected component so +# _largest_component() does not trim it. +GRAPH_AGENTS = ["su", "wiseman", "grotjahn", "ward", "briney"] +GRAPH_EDGES = [ + ("su", "wiseman"), + ("su", "grotjahn"), + ("wiseman", "ward"), + ("grotjahn", "briney"), +] +# Inside the Cabo retreat window (Apr 27 - May 7 2026) that /cabo-graph slices +# on, and after CABO_WINDOW_START (Mar 1 2026) which bounds /scripps-graph. +POST_AT = datetime(2026, 4, 28, 12, 0, tzinfo=UTC) +DECIDED_AT = datetime(2026, 4, 29, 12, 0, tzinfo=UTC) + +# Guard rail: the production database is at alembic 0018 and has real users. +ALLOWED_DATABASES = ("copi_slack_test", "copi_test", "copi_e2e") + + +def _assert_safe_database() -> str: + url = os.environ.get("DATABASE_URL", "") + name = url.rsplit("/", 1)[-1].split("?")[0] + if name not in ALLOWED_DATABASES: + sys.exit( + f"refusing to seed database {name!r}: not in {ALLOWED_DATABASES}. " + "Set DATABASE_URL to the e2e database." + ) + return name + + +async def _get_or_create_user(session, orcid, *, name, email, **kw): + from src.models import User + + row = ( + await session.execute(select(User).where(User.orcid == orcid)) + ).scalar_one_or_none() + if row: + return row, False + row = User(orcid=orcid, name=name, email=email, **kw) + session.add(row) + await session.flush() + return row, True + + +async def _get_or_create_agent(session, agent_id, **kw): + from src.models import AgentRegistry + + row = ( + await session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == agent_id) + ) + ).scalar_one_or_none() + if row: + return row, False + row = AgentRegistry(agent_id=agent_id, **kw) + session.add(row) + await session.flush() + return row, True + + +async def seed(session) -> dict[str, str]: + """Create every fixture row. Returns a summary keyed by flow.""" + from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + Job, + ResearcherProfile, + SimulationRun, + ThreadDecision, + ) + + out: dict[str, str] = {} + + admin, _ = await _get_or_create_user( + session, + ADMIN_ORCID, + name="E2E Admin", + email=ADMIN_EMAIL, + institution="Scripps Research", + is_admin=True, + access_status="allowed", + onboarding_complete=True, + ) + admin.is_admin = True # repair a row that predates the flag + out["admin_user_id"] = str(admin.id) + + # Self-service signup needs a completed profile and no agent of its own. + # "Quinn Probesmith" -> last name "probesmith" -> agent_id "probesmith"; + # no collision, so the control half of the wu/pwu rule is what we observe. + signup, created = await _get_or_create_user( + session, + SIGNUP_ORCID, + name="Quinn Probesmith", + email=SIGNUP_EMAIL, + institution="Scripps Research", + access_status="allowed", + onboarding_complete=True, + ) + if created: + session.add( + ResearcherProfile( + user_id=signup.id, + research_summary="Studies chemical probes of protein function.", + techniques=["mass spectrometry", "chemoproteomics"], + keywords=["covalent probes", "target ID"], + private_profile_md="# Private\nE2E fixture.", + profile_version=1, + ) + ) + out["signup_user_id"] = str(signup.id) + + # The onboarding walk deliberately gets NO profile and NO job, so + # /onboarding renders its first ("Building Your Profile") state. + onboarding, _ = await _get_or_create_user( + session, + ONBOARDING_ORCID, + name="Robin Onboard", + email=ONBOARDING_EMAIL, + institution="Scripps Research", + access_status="allowed", + onboarding_complete=False, + ) + # ...and RESET the row if a previous run walked it. Get-or-create alone does + # not deliver the state the paragraph above promises, because the flow is + # destructive to its own fixture: its last step POSTs + # /onboarding/private-profile, which sets onboarding_complete=True, and its + # "substitute" step leaves a ResearcherProfile and a generate_profile job in + # status 'completed' behind. Any of the three and the flow is unreplayable — + # `onboarding_complete` makes /onboarding 302 straight to /profile + # (src/routers/onboarding.py::onboarding_start, first statement), and a + # 'completed' job takes profile_review.html past the spinner branch. Measured + # on copi_slack_test 2026-08-04: all three were set from the 2026-07-31 run, + # so the flow had silently stopped testing anything a browser would see. + # Scoped to this one fixture ORCID; nothing else is deleted anywhere here. + onboarding.onboarding_complete = False + await session.execute( + delete(ResearcherProfile).where(ResearcherProfile.user_id == onboarding.id) + ) + await session.execute( + delete(Job).where(Job.user_id == onboarding.id, Job.type == "generate_profile") + ) + out["onboarding_user_id"] = str(onboarding.id) + + probe, _ = await _get_or_create_agent( + session, + PROBE_AGENT_ID, + bot_name=PROBE_BOT_NAME, + pi_name="T12 Probe", + status="pending", + ) + out["probe_agent_row_id"] = str(probe.id) + out["probe_agent_id"] = probe.agent_id + + # --- graph fixture -------------------------------------------------- + run = ( + await session.execute(select(SimulationRun).limit(1)) + ).scalar_one_or_none() + if run is None: + run = SimulationRun(status="completed", config={"seed": "tests.e2e.seed"}) + session.add(run) + await session.flush() + out["simulation_run_id"] = str(run.id) + + channel = ( + await session.execute( + select(AgentChannel).where(AgentChannel.channel_name == "e2e-general") + ) + ).scalar_one_or_none() + if channel is None: + channel = AgentChannel( + simulation_run_id=run.id, + channel_id="C0E2E0001", + channel_name="e2e-general", + channel_type="thematic", + created_by_agent=GRAPH_AGENTS[0], + ) + session.add(channel) + await session.flush() + + for i, agent_id in enumerate(GRAPH_AGENTS): + user, _ = await _get_or_create_user( + session, + f"0000-0002-0000-91{i:02d}", + name=f"PI {agent_id.title()}", + email=f"e2e-{agent_id}@example.org", + institution="Scripps Research", + access_status="allowed", + onboarding_complete=True, + ) + agent = ( + await session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == agent_id) + ) + ).scalar_one_or_none() + if agent is None: + session.add( + AgentRegistry( + agent_id=agent_id, + user_id=user.id, + bot_name=f"{agent_id.title()}Bot", + pi_name=f"PI {agent_id.title()}", + status="active", + ) + ) + await session.flush() + + for i, (a, b) in enumerate(GRAPH_EDGES): + ts = f"17{i:08d}.000100" + existing = ( + await session.execute( + select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.message_ts == ts, + ) + ) + ).scalar_one_or_none() + if existing is None: + session.add( + AgentMessage( + simulation_run_id=run.id, + agent_id=a, + channel_id=channel.channel_id, + channel_name=channel.channel_name, + message_ts=ts, + phase="new_post", + visibility="public", + content=f"{a} proposes work with {b}.", + sender_name=f"{a.title()}Bot", + message_length=40, + posted_at=POST_AT.timestamp(), + created_at=POST_AT, + ) + ) + decided = ( + await session.execute( + select(ThreadDecision).where(ThreadDecision.thread_id == ts) + ) + ).scalar_one_or_none() + if decided is None: + session.add( + ThreadDecision( + simulation_run_id=run.id, + thread_id=ts, + channel=channel.channel_name, + agent_a=a, + agent_b=b, + outcome="proposal", + origin_visibility="public", + summary_text=( + f"{a.title()} and {b.title()} propose a joint study " + "combining their platforms." + ), + decided_at=DECIDED_AT, + ) + ) + await session.commit() + out["graph_edges"] = str(len(GRAPH_EDGES)) + return out + + +async def main() -> None: + name = _assert_safe_database() + from src.database import get_session_factory + + async with get_session_factory()() as session: + summary = await seed(session) + print(f"seeded {name}:") + for k, v in summary.items(): + print(f" {k} = {v}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/e2e/session.py b/tests/e2e/session.py new file mode 100644 index 0000000..f513d30 --- /dev/null +++ b/tests/e2e/session.py @@ -0,0 +1,36 @@ +"""Forge the signed session cookie ``SessionMiddleware`` would issue. + +Identical construction to ``tests/integration/test_cohort_admin.py::_auth`` — +``itsdangerous.TimestampSigner(secret_key)`` over ``base64(json(session))``, +under cookie name ``copi-session`` (see ``src/main.py``). Kept in its own module +so both the pytest flows and the host-side ``auth_helper`` can use it. + +This is how the browser flows authenticate. It is a deliberate bypass of ORCID +login, which is broken in this deployment (root cause in ``README.md``), not a +convenience: the flows under test are the *admin* and *agent* surfaces, and +holding them hostage to a third-party OAuth outage would test ORCID rather than +us. It requires the signing key, so it is no weaker than the deployment already +is. +""" + +import base64 +import json + +from itsdangerous import TimestampSigner + +COOKIE_NAME = "copi-session" + + +def forge_session_cookie(user_id: str, **extra) -> str: + """Return the cookie *value* for a session holding ``user_id``.""" + from src.config import get_settings + + signer = TimestampSigner(get_settings().secret_key) + payload = {"user_id": str(user_id), **extra} + data = base64.b64encode(json.dumps(payload).encode()) + return signer.sign(data).decode() + + +def auth_headers(user_id: str, **extra) -> dict[str, str]: + """Request headers carrying a forged session for ``user_id``.""" + return {"Cookie": f"{COOKIE_NAME}={forge_session_cookie(user_id, **extra)}"} diff --git a/tests/e2e/test_browser_flows.py b/tests/e2e/test_browser_flows.py new file mode 100644 index 0000000..eb0e6d6 --- /dev/null +++ b/tests/e2e/test_browser_flows.py @@ -0,0 +1,485 @@ +"""Task 12 — browser flows, as repeatable scripts. + +Two layers, because the two things worth recording are different: + +1. **``FLOWS``** — a machine-readable transcript of every flow: what to open, + what to click, and what must be visible. This is the part a human (or an + MCP-driven browser agent) replays. Playwright-over-MCP is interactive, so the + plan (.notes/full-system-test-plan.md, Task 12) asks for scripts rather than + pytest tests; ``FLOWS`` is that script, and ``test_every_flow_is_well_formed`` + keeps it honest. + +2. **HTTP replays** — for every flow whose steps are ordinary form posts, a + pytest test drives the *same* route sequence against the running server with + ``httpx`` and asserts the same visible strings. Those have teeth without a + browser and are what you run in anger. They are skipped unless + ``E2E_BASE_URL`` is set, because they need a live server on a migrated + database — see ``tests/e2e/README.md`` for the two-command setup. + +Authentication is a forged session cookie (``tests/e2e/session.py``), not ORCID +login. ORCID login is *broken in this deployment* and the root cause is in +``README.md``; holding the admin and agent surfaces hostage to it would test +ORCID rather than us. + +What cannot be automated at all, and why, is in ``HUMAN_ONLY``. +""" + +import os +import re +import uuid + +import httpx +import pytest + +from tests.e2e.session import COOKIE_NAME, forge_session_cookie + +BASE_URL = os.environ.get("E2E_BASE_URL", "") +# A second instance of the same app on the same database with +# COHORT_ISOLATION_ENABLED=true (and optionally COHORT_DEFAULT_POLICY=isolated). +# Needed for the banner control: cohort settings are read once per process, so +# proving the banner tracks them requires a second process, not a second request. +ISOLATION_URL = os.environ.get("E2E_ISOLATION_BASE_URL", "") + +requires_server = pytest.mark.skipif( + not BASE_URL, + reason="needs E2E_BASE_URL pointing at a running app on a migrated database", +) +requires_isolation_server = pytest.mark.skipif( + not ISOLATION_URL, + reason="needs E2E_ISOLATION_BASE_URL (same app, COHORT_ISOLATION_ENABLED=true)", +) + + +# --------------------------------------------------------------------------- +# The scripts +# --------------------------------------------------------------------------- + +#: Each flow: ``steps`` are (action, target, note); ``expect`` are substrings +#: that MUST appear in the rendered page at the end of the flow. +FLOWS: dict[str, dict] = { + "admin_cohort_and_topology": { + "as": "admin", + "human_needed": False, + "steps": [ + ("open", "/admin/cohorts", "banner must state the LIVE setting"), + ("click", "New Cohort", "reveals the inline create form"), + ("fill", "name=t12-browser-flow", "lowercase/hyphen only, max 48"), + ("fill", "description=Created by the Task 12 browser flow", ""), + ("click", "Create Cohort", "302s to /admin/cohorts/{id}"), + ("open", "/admin/cohorts/topology", "agent x cohort matrix"), + ("check", "cell SuBot x t12-browser-flow", ""), + ("check", "cell WisemanBot x t12-browser-flow", ""), + ("click", "Save topology", "302s with ?notice=2+added,+0+removed"), + ], + "expect": [ + "2 added, 0 removed", + "Cohort isolation is OFF", + "everyone (gate off for this agent)", + ], + "control": ( + "Repeat the last open against a process started with " + "COHORT_ISOLATION_ENABLED=true: the banner must read 'Cohort " + "isolation is ACTIVE' and SuBot's 'Acts on' cell must stop saying " + "'gate off'. Without this half, the banner could be static text." + ), + }, + "agent_self_service_signup": { + "as": "signup", + "human_needed": False, + "steps": [ + ("open", "/agent", "no agent yet -> request page"), + ("click", "Request Agent", "POST /agent/request"), + ], + "expect": [ + "Agent Request Pending", + "ProbesmithBot", + ], + "control": ( + "'Quinn Probesmith' has no last-name collision, so the agent_id is " + "unprefixed 'probesmith'. The collision half of the rule (wu -> " + "pwu) is a unit-level concern; assert it in " + "tests/integration/test_agent_page.py, not here." + ), + }, + "public_graph": { + "as": None, # unauthenticated on purpose: these routes take no auth + "human_needed": False, + "steps": [ + ("open", "/scripps-graph", "D3 force layout over real rows"), + ("click", "an edge", "opens the proposal modal"), + ], + "expect": [ + "Scripps Research collaboration network", + "5 Scripps PIs", + "4 collaborating pairs", + "4 joint proposals", + ], + "control": ( + "The counts come from tests.e2e.seed's 5 agents / 4 edges. A route " + "that rendered an empty graph would still return 200, so the " + "assertion is on the counts, not on the status code." + ), + }, + "onboarding": { + "as": "onboarding", + "human_needed": False, + "stops_at": ( + "Step 3 of 4, 'Building Your Profile'. /onboarding auto-enqueues a " + "generate_profile job and the template shows that spinner for " + "job_status in (none, pending, processing). Completing the step " + "needs the worker to run run_profile_pipeline, which fetches the " + "user's ORCID record — so with no usable ORCID credentials it can " + "never finish. The steps below therefore substitute the profile row " + "the pipeline would have written; the pipeline itself is Task 4's " + "subject, not this one." + ), + "steps": [ + ("open", "/onboarding", "Step 3 of 4 spinner, job enqueued"), + ("substitute", "ResearcherProfile + jobs.status='completed'", + "stands in for the ORCID-fed pipeline"), + ("open", "/onboarding", "now renders the editable review form"), + ("click", "Save & Continue", "POST /onboarding/save-profile"), + # The button lives in private_profile.html and always posted here; + # this note said POST /onboarding/complete, which was wrong even + # before that duplicate route was deleted for setting + # onboarding_complete with no validation. + ("click", "Save & Complete Onboarding", "POST /onboarding/private-profile"), + ], + "expect": [ + "onboarding_complete=1", + ], + "control": ( + "Assert users.onboarding_complete was FALSE at /onboarding and TRUE " + "only after the final POST. Otherwise a route that set the flag on " + "first view would pass." + ), + }, + "slack_provisioning": { + "as": "admin", + "human_needed": True, + "steps": [ + ("arm", "SLACK_CONFIG_REFRESH_TOKEN on the app process", + "single-use; the first click spends it"), + ("open", "/admin/agents/{probe_agent_row_id}", ""), + ("click", "Provision", + "POST .../slack/provision -> apps.manifest.create -> 302 to Slack"), + ("human", "Allow, on Slack's install screen", + "the automation browser has no Slack session"), + ("land", "/admin/agents/{id}?slack_ok=1", + "Slack redirects to BASE_URL/admin/agents/slack/callback"), + ("verify", "auth.test on the resulting xoxb- token", + "read back from Slack, never from our own column"), + ], + "expect": [ + "Slack bot provisioned", + ], + "control": ( + "A set slack_bot_token column proves only that we wrote a column. " + "The assertion is auth.test returning ok=true with the expected " + "team_id and a bot_id — i.e. Slack agrees the app exists and is " + "installed." + ), + }, +} + +#: Flows that cannot run headless, with the reason. Kept as data so +#: ``tests/e2e/README.md`` and this module cannot drift apart. +HUMAN_ONLY = { + "slack_provisioning": ( + "Slack's OAuth consent screen requires a browser with a Slack session. " + "The Playwright/MCP browser has none and there is no API to obtain one " + "(Slack has no headless install grant). Everything either side of the " + "Allow click is automated: the Provision POST is driven by the test, " + "and the callback is the app's own route." + ), + "orcid_login": ( + "Not in FLOWS at all: it cannot be driven end to end from any browser " + "in this deployment. ORCID rejects the configured client_id (HTTP 400 " + "invalid_request / 'Invalid parameter: client_id'), so there is no " + "consent screen to click. See README.md." + ), +} + + +# --------------------------------------------------------------------------- +# Script well-formedness (runs offline — this is the part that stops FLOWS +# from rotting into prose) +# --------------------------------------------------------------------------- + +_ACTIONS = { + "open", "click", "fill", "check", "human", "land", "verify", "arm", + "substitute", +} + + +def test_every_flow_is_well_formed(): + assert FLOWS, "no flows recorded" + for name, flow in FLOWS.items(): + assert flow["steps"], f"{name}: no steps" + assert flow["expect"], f"{name}: nothing asserted visible" + assert flow.get("control"), f"{name}: no control stated" + assert "human_needed" in flow, f"{name}: does it need a human?" + for action, target, _note in flow["steps"]: + assert action in _ACTIONS, f"{name}: unknown action {action!r}" + assert target, f"{name}: empty target for {action!r}" + + +def test_human_only_matches_the_flows(): + """Every human_needed flow is explained in HUMAN_ONLY, and vice versa. + + Control: HUMAN_ONLY also carries ``orcid_login``, which is deliberately not + a flow — so this asserts a subset relation in one direction only, and the + ``needs`` check below is what actually has teeth. + """ + needs = {n for n, f in FLOWS.items() if f["human_needed"]} + assert needs == {"slack_provisioning"}, needs + assert needs <= set(HUMAN_ONLY), needs - set(HUMAN_ONLY) + + +# --------------------------------------------------------------------------- +# HTTP replays against a live server +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def admin_id() -> str: + """Admin user id. Provided by the harness, since the forged cookie needs it + before any authenticated request can be made.""" + value = os.environ.get("E2E_ADMIN_USER_ID", "") + if not value: + pytest.skip("needs E2E_ADMIN_USER_ID (printed by python -m tests.e2e.seed)") + uuid.UUID(value) # fail loudly on a malformed id rather than at the route + return value + + +@pytest.fixture +def client(): + """An anonymous client, FRESH per test. + + Deliberately not module-scoped: ``SessionMiddleware`` re-issues + ``copi-session`` on every response that carries a session, so a shared + client accumulates one identity's cookie and the next test silently runs as + that identity. + """ + with httpx.Client(base_url=BASE_URL, follow_redirects=True, timeout=30) as c: + yield c + + +@pytest.fixture +def as_user(): + """``as_user(user_id) -> httpx.Client`` authenticated as that user. + + The forged cookie goes into the **cookie jar**, not into a per-request + ``Cookie`` header. That distinction is load-bearing and cost two red tests: + + * the app re-signs and re-sets ``copi-session`` on every response, so the + jar fills up during the flow; + * httpx then sends the jar cookie *in addition to* an explicit ``Cookie`` + header, producing two ``Cookie`` header values. Starlette reads + ``headers.get("cookie")``, which joins them with ", ", and the resulting + ``copi-session=A, copi-session=B`` fails to parse — so the request arrives + **unauthenticated**. + * that only bites on a redirect-follow (302 -> GET), i.e. exactly on the + form posts, which is why the GETs looked fine. + + Using the jar is also what a browser actually does. + """ + created: list[httpx.Client] = [] + + def _make(user_id: str) -> httpx.Client: + c = httpx.Client(base_url=BASE_URL, follow_redirects=True, timeout=30) + c.cookies.set(COOKIE_NAME, forge_session_cookie(user_id)) + created.append(c) + return c + + yield _make + for c in created: + c.close() + + +@requires_server +def test_public_graph_renders_with_real_data(client): + """FLOWS['public_graph']. + + Rule L3-style attribution: each assertion says which failure it saw. + """ + r = client.get("/scripps-graph") + assert r.status_code == 200, f"/scripps-graph did not render: {r.status_code}" + for want in FLOWS["public_graph"]["expect"]: + assert want in r.text, ( + f"{want!r} missing from /scripps-graph — either the seed is absent " + "(run python -m tests.e2e.seed) or the graph query stopped matching " + "the seeded rows" + ) + # Control: an empty graph would also be a 200, so assert the payload has + # nodes AND that a proposal summary reached the page (the modal's content). + assert '"nodes": [{' in r.text, "graph payload has no nodes" + assert "propose a joint study" in r.text, ( + "no proposal summary in the payload: thread_decisions did not join to " + "the in-window new_post messages" + ) + + +@requires_server +def test_admin_cohort_create_and_topology_edit(as_user, admin_id): + """FLOWS['admin_cohort_and_topology'] — create, then edit the matrix. + + Idempotent: the cohort name is reused, and a second run sees + "already exists" and still exercises the topology save. + """ + c = as_user(admin_id) + + r = c.get("/admin/cohorts") + assert r.status_code == 200, "admin cohort list is not reachable" + assert "Cohort isolation is" in r.text, "the gate banner is missing entirely" + + c.post( + "/admin/cohorts/create", + data={"name": "t12-browser-flow", "description": "Task 12 browser flow"}, + ) + + r = c.get("/admin/cohorts/topology") + assert r.status_code == 200, "topology matrix is not reachable" + cells = re.findall(r'name="present" value="([0-9a-f-]{36}:[a-z0-9]+)"', r.text) + assert cells, "the matrix rendered no cells — no cohorts or no agents seeded" + wanted = [x for x in cells if x.endswith((":su", ":wiseman"))] + assert len(wanted) == 2, f"expected su and wiseman cells, got {wanted}" + + # httpx wants a dict-of-lists for repeated form keys; a list of 2-tuples is + # sent as raw content and h11 rejects it. + r = c.post("/admin/cohorts/topology", data={"present": cells, "cell": wanted}) + assert r.status_code == 200 + assert "added," in r.text and "removed" in r.text, ( + "the save did not report a diff — either the form shape changed or the " + f"redirect landed somewhere else: {r.url}" + ) + # Both halves: the memberships are now reflected back in the rendered form. + # `checked` is several attributes after `value` in the template, so match the + # whole element rather than assuming attribute order. + r = c.get("/admin/cohorts/topology") + checked = [ + m.group(1) + for m in re.finditer( + r'name="cell" value="([0-9a-f-]{36}:[a-z0-9]+)"([^>]*)>', r.text + ) + if "checked" in m.group(2) + ] + assert set(checked) == set(wanted), ( + f"saved memberships not reflected on reload: {checked} != {wanted}" + ) + + +@requires_server +def test_the_banner_states_the_live_setting_not_a_constant(as_user, admin_id): + """The control for the banner. Two processes, two settings, two banners. + + Without this, ``Cohort isolation is OFF`` passing proves nothing: a template + that hardcoded it would pass too. + """ + off = as_user(admin_id).get("/admin/cohorts").text + assert "Cohort isolation is OFF" in off, ( + "the default process should report isolation OFF; if this fails, " + "COHORT_ISOLATION_ENABLED leaked into the E2E_BASE_URL process" + ) + assert "Cohort isolation is ACTIVE" not in off + + +@requires_server +@requires_isolation_server +def test_the_banner_and_gate_change_when_isolation_is_on(admin_id): + """Second half of the control, against the isolation-on process.""" + with httpx.Client(base_url=ISOLATION_URL, follow_redirects=True, timeout=30) as c: + c.cookies.set(COOKIE_NAME, forge_session_cookie(admin_id)) + page = c.get("/admin/cohorts/topology").text + assert "Cohort isolation is ACTIVE" in page, ( + "the isolation-on process still reports OFF — E2E_ISOLATION_BASE_URL is " + "probably pointing at the same process as E2E_BASE_URL" + ) + assert "active agents are gated" in page + # The gate preview must stop saying "gate off" for the agents we put in the + # cohort. That is the assertion with teeth: the banner alone could be text. + assert page.count("everyone (gate off for this agent)") < 5, ( + "isolation is reported ACTIVE but every agent is still ungated — the " + "topology saved by the previous test is not reaching compute_gates" + ) + + +@requires_server +def test_agent_self_service_signup(as_user): + """FLOWS['agent_self_service_signup']. + + Needs E2E_SIGNUP_USER_ID. Idempotent: a second run finds the agent already + present and still lands on the pending page. + """ + user_id = os.environ.get("E2E_SIGNUP_USER_ID", "") + if not user_id: + pytest.skip("needs E2E_SIGNUP_USER_ID (printed by python -m tests.e2e.seed)") + + r = as_user(user_id).post("/agent/request") + assert r.status_code == 200, f"POST /agent/request failed: {r.status_code}" + for want in FLOWS["agent_self_service_signup"]["expect"]: + assert want in r.text, ( + f"{want!r} missing after signup — either the request was rejected " + "(the user needs onboarding_complete AND a profile) or the pending " + "template changed" + ) + # Control: the derived bot name comes from the user's surname, so a wrong + # derivation (e.g. using the first name) would still render a pending page. + assert "ProbesmithBot" in r.text + + +@requires_server +def test_onboarding_goes_as_far_as_the_orcid_dependency(as_user): + """FLOWS['onboarding'] — the honest stopping point. + + Asserts the spinner, NOT the completion: a fresh onboarding user cannot get + past step 3 without a profile, and a profile cannot be built without a + usable ORCID record. See FLOWS['onboarding']['stops_at']. + """ + user_id = os.environ.get("E2E_ONBOARDING_USER_ID", "") + if not user_id: + pytest.skip("needs E2E_ONBOARDING_USER_ID (printed by tests.e2e.seed)") + r = as_user(user_id).get("/onboarding") + assert r.status_code == 200, "/onboarding is not reachable with a session" + if "Building Your Profile" in r.text: + assert "Step 3 of 4" in r.text + return + # Otherwise the fixture was walked to completion by a previous run, and + # /onboarding 302s to /profile. Both outcomes are correct; which one we saw + # is the Rule L3 part — so assert it on the LANDING PATH. + # + # This assertion used to be `"Profile" in r.text`, which discriminates + # nothing: the spinner page above renders the literal string "Building Your + # Profile", so it satisfies this branch too, and so does any other page + # carrying the app's nav. A route that had stopped honouring + # onboarding_complete would have passed it. The path is the one observable + # that differs between the two documented states. + # + # Reaching this branch at all now means tests.e2e.seed's onboarding reset did + # not run — re-seed before believing anything this test says. + assert r.url.path == "/profile", ( + "/onboarding neither showed the pipeline spinner nor redirected to the " + f"completed profile — the flow is in neither documented state: {r.url}" + ) + + +@requires_server +def test_orcid_login_cannot_be_driven_and_says_why(client): + """Not a flow: a pin on the finding, so it cannot be silently 'fixed'. + + /login/start must 302 to orcid.org with the configured client_id. If that + client_id is a placeholder, ORCID answers 400 and there is no consent + screen — which is why every other flow here forges a cookie instead. + """ + r = client.get("/login/start", follow_redirects=False) + assert r.status_code == 302, "login/start no longer redirects" + location = r.headers["location"] + assert location.startswith("https://orcid.org/oauth/authorize"), location + m = re.search(r"client_id=([^&]+)", location) + assert m, f"no client_id in the ORCID authorize URL: {location}" + if m.group(1) in ("test-client-id", ""): + pytest.xfail( + "ORCID_CLIENT_ID is the placeholder 'test-client-id': ORCID answers " + "HTTP 400 invalid_request / 'Invalid parameter: client_id', so no " + "browser can complete login. Configure real ORCID OAuth credentials." + ) diff --git a/tests/fakes.py b/tests/fakes.py index 5668614..dede2d6 100644 --- a/tests/fakes.py +++ b/tests/fakes.py @@ -128,10 +128,6 @@ def __init__(self, agent_id: str = "agent1", bot_token: str = "xoxb-fake") -> No self.created_channels: list[dict] = [] self.invites: list[dict] = [] self._ts = 1_700_000_000 - self._visibility_lookup: Callable[[str], str | None] | None = None - - def set_visibility_lookup(self, lookup: Callable[[str], str | None]) -> None: - self._visibility_lookup = lookup def connect(self) -> bool: return True @@ -182,3 +178,79 @@ def invite_to_channel(self, channel_id: str, user_ids: list[str]) -> bool: def list_channels(self, include_private: bool = False) -> dict: return {} + + def _resolve_channel_id(self, channel: str) -> str: + """Name -> id, mirroring AgentSlackClient (ids pass through unchanged).""" + if channel.startswith(("C", "G")): + return channel + return f"C_{channel}" + + +class RecordingSlackClient: + """Records outbound Slack Web API calls; scripts responses and errors. + + Deliberately distinct from ``FakeSlackClient``. That one implements the + *Transport* protocol and returns canned values, which means a mirror that never + called Slack at all would satisfy it just as well as one that did. This class + stands in for the ``slack_sdk.WebClient`` **inside** ``AgentSlackClient``, and + ``.calls`` is the evidence that the call was made and with what arguments. + + ``responses`` maps a WebClient method name to the dict it should return. + ``errors`` maps a method name to a list of exceptions, popped one per call, so a + retry path can be scripted as "fail, then succeed". + """ + + def __init__(self, responses=None, errors=None): + self.calls: list[tuple[str, dict]] = [] + self._responses = dict(responses or {}) + self._errors = {k: list(v) for k, v in (errors or {}).items()} + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + + def _call(**kwargs): + self.calls.append((name, kwargs)) + queue = self._errors.get(name) + if queue: + raise queue.pop(0) + return _SlackResponse(self._responses.get(name, {"ok": True})) + + return _call + + def calls_to(self, method: str) -> list[dict]: + return [kw for m, kw in self.calls if m == method] + + +class _SlackResponse: + """The parts of slack_sdk's SlackResponse that AgentSlackClient touches.""" + + def __init__(self, data: dict): + self.data = data + self.headers: dict[str, str] = {} + self.status_code = 200 + + def get(self, key, default=None): + return self.data.get(key, default) + + def __getitem__(self, key): + return self.data[key] + + def __contains__(self, key): + return key in self.data + + +def slack_error(code: str, *, retry_after: int | None = None): + """A SlackApiError shaped the way ``_call_with_retry`` inspects it. + + It reads ``exc.response.get("error")`` and ``exc.response.headers.get("Retry-After")``, + so both have to be present on the response object or the retry branch is never + reached and the test would pass for the wrong reason. + """ + from slack_sdk.errors import SlackApiError + + resp = _SlackResponse({"ok": False, "error": code}) + if retry_after is not None: + resp.headers = {"Retry-After": str(retry_after)} + resp.status_code = 429 if code == "ratelimited" else 400 + return SlackApiError(code, resp) diff --git a/tests/integration/test_agent_page.py b/tests/integration/test_agent_page.py new file mode 100644 index 0000000..22edf63 --- /dev/null +++ b/tests/integration/test_agent_page.py @@ -0,0 +1,1300 @@ +"""Live integration tests for the agent page — all 19 endpoints of routers/agent_page.py. + +Real ASGI requests, real Postgres, real Jinja templates, real invitation/reopen +flows. Task T8 of .notes/full-system-test-plan.md. + +Nothing external is real: Slack (`slack_sdk.WebClient` and the copy bound inside +`src.agent.slack_client`), SES (`send_delegate_invitation`) and the whole httpx +transport layer are replaced by recorders that fail loudly if a test reaches for +the network. The database is NOT mocked — every assertion below is about rows +the routes actually wrote. + +Discipline (see the plan's "The Discipline"): + * every absence assertion carries a positive control in the same test; + * state is produced by driving the real route, not hand-built, wherever the + route that produces it is itself under test; + * two known defects are pinned with ``xfail(strict=True)`` so the suite goes + red the day they are fixed and the assertion has to be flipped, instead of + quietly encoding a bug as expected behaviour. +""" + +import base64 +import json +import re +import uuid +from dataclasses import dataclass, field +from types import SimpleNamespace +from urllib.parse import unquote + +import pytest +from itsdangerous import TimestampSigner +from sqlalchemy import select + +from src.config import get_settings +from src.models import ( + VISIBILITY_COLLAB_PRIVATE, + AgentChannel, + AgentDelegate, + AgentMessage, + AgentRegistry, + DelegateInvitation, + PiDmMessage, + PrivateChannelMember, + ProfileRevision, + ProposalReview, + ResearcherProfile, +) +from tests import factories + +pytestmark = pytest.mark.integration + + +def _auth(user_id) -> dict: + """Forge the signed session cookie SessionMiddleware would issue.""" + signer = TimestampSigner(get_settings().secret_key) + data = base64.b64encode(json.dumps({"user_id": str(user_id)}).encode()) + return {"Cookie": f"copi-session={signer.sign(data).decode()}"} + + +# --------------------------------------------------------------------------- +# External-world doubles. None of these may reach the network. +# --------------------------------------------------------------------------- + + +class _SlackRecorder: + """Records every Slack API call the routes attempt. + + Unstubbed methods raise: a route that starts talking to Slack where these + tests assert it must not will blow up rather than silently succeed against + a permissive Mock. `calls` is asserted to be empty in the reopen tests — + that is the "never make a live Slack call" constraint, enforced. + """ + + def __init__(self): + self.calls: list[tuple[str, dict]] = [] + self.handlers: dict[str, object] = {} + + def stub(self, method: str, response): + self.handlers[method] = response + + @property + def methods(self) -> list[str]: + return [name for name, _ in self.calls] + + +class _FakeWebClient: + def __init__(self, recorder: _SlackRecorder, **kwargs): + self._rec = recorder + self.token = kwargs.get("token") + + def __getattr__(self, name): + def _call(**kwargs): + self._rec.calls.append((name, kwargs)) + if name not in self._rec.handlers: + raise AssertionError( + f"unstubbed Slack API call {name}({kwargs}) — tests must never " + "reach the real workspace" + ) + resp = self._rec.handlers[name] + return resp(**kwargs) if callable(resp) else resp + + return _call + + +@pytest.fixture(autouse=True) +def slack(monkeypatch) -> _SlackRecorder: + rec = _SlackRecorder() + factory = lambda *a, **kw: _FakeWebClient(rec, **kw) # noqa: E731 + monkeypatch.setattr("slack_sdk.WebClient", factory) + # AgentSlackClient bound WebClient at import time, so patch that name too — + # it is the one the private-channel migration would use. + monkeypatch.setattr("src.agent.slack_client.WebClient", factory) + # services/slack_web.py is the web layer's Slack boundary and binds WebClient + # at import time as well. Patching only `slack_sdk.WebClient` would leave the + # routes' user lookups and channel listing talking to the real workspace. + monkeypatch.setattr("src.services.slack_web.WebClient", factory) + return rec + + +@pytest.fixture(autouse=True) +def _slack_enabled_auto_detect(monkeypatch): + """Hermetic default for the Slack on/off tri-state (src/services/slack_tokens.py + and src/services/private_channels.py's ``_slack_enabled_for_migration``): unset + (auto-detect from token presence) rather than whatever ``SLACK_ENABLED`` the + deployed .env on this host forces. + + Without this, a populated .env with SLACK_ENABLED=true forces the real-Slack + branch of `reopen_proposal` even for `world`'s fictitious agents (`tstowner`, + `tstother`), which have no token anywhere — `migrate_public_thread_to_private` + then 500s on "No valid Slack bot token". Auto-detect is this suite's actual + premise: a test that wants Slack ON gives its own agent a token (e.g. + ``world.agent.slack_bot_token = "xoxb-fake-for-tests"``), which is what + auto-detect keys on either way. + """ + monkeypatch.setattr(get_settings(), "slack_enabled", None) + + +@pytest.fixture(autouse=True) +def sent_emails(monkeypatch) -> list[dict]: + """Recording double for the SES leg (the plan's email seam: record, never send).""" + sent: list[dict] = [] + + def _send(to_email, pi_name, bot_name, invite_url): + sent.append( + {"to": to_email, "pi_name": pi_name, "bot_name": bot_name, "url": invite_url} + ) + return True + + monkeypatch.setattr("src.services.email.send_delegate_invitation", _send) + return sent + + +@pytest.fixture(autouse=True) +def no_network(monkeypatch): + """Any real outbound HTTP (Anthropic, ORCID, NCBI, grants.gov) is a test bug. + + Patched at the transport layer, which the ASGI test client does not use, so + in-process requests still work. + """ + + def _boom(*args, **kwargs): + raise AssertionError("a test attempted a real outbound HTTP request") + + monkeypatch.setattr("httpx.HTTPTransport.handle_request", _boom) + monkeypatch.setattr("httpx.AsyncHTTPTransport.handle_async_request", _boom) + + +@pytest.fixture(autouse=True) +def profiles_dir(tmp_path, monkeypatch): + """Keep the profile-save routes off the repo's real profiles/ directory.""" + monkeypatch.setattr("src.routers.agent_page.PROFILES_DIR", tmp_path / "profiles") + monkeypatch.setattr( + "src.services.profile_export.PROFILES_DIR", tmp_path / "profiles" / "public" + ) + monkeypatch.setattr( + "src.services.profile_export.PRIVATE_PROFILES_DIR", tmp_path / "profiles" / "private" + ) + return tmp_path / "profiles" + + +# --------------------------------------------------------------------------- +# World fixtures +# --------------------------------------------------------------------------- + +# Deliberately not real roster slugs (`su`, `wu`, …): config.get_slack_tokens() +# is keyed by those, and a populated .env would hand the migration path a real +# bot token. +OWNER_AGENT = "tstowner" +OTHER_AGENT = "tstother" +THIRD_AGENT = "tstthird" + + +async def _agent_for(db, *, name, email, agent_id, bot_name, status="active"): + user = await factories.make_user(db, name=name, email=email) + agent = await factories.make_agent( + db, user=user, agent_id=agent_id, bot_name=bot_name, pi_name=name, status=status + ) + return user, agent + + +@pytest.fixture +async def world(db_session): + """One owned active agent, a counterpart agent, a stranger, a run and a proposal.""" + pi, agent = await _agent_for( + db_session, name="Pat Owner", email="pi@example.org", + agent_id=OWNER_AGENT, bot_name="OwnerBot", + ) + await factories.make_profile(db_session, user=pi) + other_pi, other_agent = await _agent_for( + db_session, name="Otto Other", email="otto@example.org", + agent_id=OTHER_AGENT, bot_name="OtherBot", + ) + stranger = await factories.make_user( + db_session, name="Sam Stranger", email="stranger@example.org" + ) + run = await factories.make_simulation_run(db_session) + td = await factories.make_thread_decision( + db_session, run=run, agent_a=OWNER_AGENT, agent_b=OTHER_AGENT, + channel="general", outcome="proposal", + summary_text=":memo: **Summary — A shared assay platform**\nBoth labs need it.", + ) + await db_session.flush() + return SimpleNamespace( + pi=pi, agent=agent, other_pi=other_pi, other_agent=other_agent, + stranger=stranger, run=run, td=td, + ) + + +async def _invite(client, world, email): + """Drive the real invite route; return the DelegateInvitation token.""" + r = await client.post( + f"/agent/{OWNER_AGENT}/delegates/invite", + data={"emails": email}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302, r.text + assert "delegate_error" not in r.headers["location"], r.headers["location"] + return r + + +async def _token_for(db, agent, email) -> str: + row = (await db.execute( + select(DelegateInvitation).where( + DelegateInvitation.agent_registry_id == agent.id, + DelegateInvitation.email == email, + ) + )).scalar_one() + return row.token + + +@pytest.fixture +async def delegated(client, db_session, world): + """A delegate produced by the real invite → accept flow, plus a pending invite. + + Built by driving the routes rather than inserting an AgentDelegate row, so + the fixture itself proves the add path works before anything asserts on it. + """ + delegate = await factories.make_user( + db_session, name="Dee Legate", email="dee@example.org" + ) + await _invite(client, world, "dee@example.org") + token = await _token_for(db_session, world.agent, "dee@example.org") + r = await client.post(f"/invite/{token}/accept", headers=_auth(delegate.id)) + assert r.status_code == 302 and r.headers["location"].endswith( + f"/agent/{OWNER_AGENT}/dashboard" + ) + row = (await db_session.execute( + select(AgentDelegate).where(AgentDelegate.agent_registry_id == world.agent.id) + )).scalar_one() + + # A second, still-pending invitation so the revoke endpoint has a real target. + await _invite(client, world, "pending@example.org") + pending = (await db_session.execute( + select(DelegateInvitation).where( + DelegateInvitation.agent_registry_id == world.agent.id, + DelegateInvitation.status == "pending", + ) + )).scalar_one() + return SimpleNamespace(user=delegate, row=row, pending_invitation=pending) + + +async def _private_channels(db) -> list[AgentChannel]: + return list((await db.execute( + select(AgentChannel) + .where(AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE) + .order_by(AgentChannel.channel_name) + )).scalars().all()) + + +async def _reviews(db, agent_id: str) -> list[ProposalReview]: + return list((await db.execute( + select(ProposalReview).where(ProposalReview.agent_id == agent_id) + )).scalars().all()) + + +# =========================================================================== +# 1. Self-service signup (POST /agent/request) +# =========================================================================== + + +async def _signup(client, db_session, name, email): + user = await factories.make_user(db_session, name=name, email=email) + await factories.make_profile(db_session, user=user) + await db_session.flush() + r = await client.post("/agent/request", headers=_auth(user.id)) + return user, r + + +async def _agent_of(db, user) -> AgentRegistry | None: + return (await db.execute( + select(AgentRegistry).where(AgentRegistry.user_id == user.id) + )).scalar_one_or_none() + + +async def test_signup_creates_a_pending_agent_row(client, db_session): + """The documented self-service path (CLAUDE.md §Adding New PIs).""" + user, r = await _signup(client, db_session, "Ada Zephyr", "ada@example.org") + assert r.status_code == 302 and r.headers["location"] == "/agent" + + agent = await _agent_of(db_session, user) + assert agent is not None, "POST /agent/request created no AgentRegistry row" + assert agent.agent_id == "zephyr" + assert agent.bot_name == "ZephyrBot" + assert agent.pi_name == "Ada Zephyr" + # Signup must not self-approve: an agent that came out 'active' would join + # the simulation roster (_sync_roster_from_db) without an admin ever looking. + assert agent.status == "pending" + assert agent.slack_bot_token is None + + +async def test_signup_prefixes_the_first_initial_only_on_a_last_name_collision( + client, db_session +): + """CLAUDE.md: "Chunlei Wu = wu"; a second Wu becomes "pwu". + + Control (the second half): a *non*-colliding last name must come out + unprefixed. Without it a request_agent() that always prefixed would pass. + """ + first, r1 = await _signup(client, db_session, "Chunlei Wu", "chunlei@example.org") + assert r1.status_code == 302 + assert (await _agent_of(db_session, first)).agent_id == "wu" + + second, r2 = await _signup(client, db_session, "Peng Wu", "peng@example.org") + assert r2.status_code == 302 + assert (await _agent_of(db_session, second)).agent_id == "pwu" + + # Control: no collision → no prefix (not "azephyr"). + control, r3 = await _signup(client, db_session, "Ada Zephyr", "ada@example.org") + assert r3.status_code == 302 + assert (await _agent_of(db_session, control)).agent_id == "zephyr" + + +async def test_signup_collision_also_disambiguates_the_bot_name(client, db_session): + await _signup(client, db_session, "Chunlei Wu", "chunlei@example.org") + second, _ = await _signup(client, db_session, "Peng Wu", "peng@example.org") + assert (await _agent_of(db_session, second)).bot_name == "PWuBot" + + +async def test_signup_needs_a_completed_profile(client, db_session): + """Absence assertion + its control, in one test.""" + bare = await factories.make_user( + db_session, name="Nora Newbie", email="nora@example.org", + onboarding_complete=False, + ) + await db_session.flush() + r = await client.post("/agent/request", headers=_auth(bare.id)) + assert r.status_code == 400 + assert await _agent_of(db_session, bare) is None + + # Control: the same request from a user who *has* finished onboarding works, + # so the 400 above is about the profile gate and not about the route. + ready, r2 = await _signup(client, db_session, "Ready Researcher", "ready@example.org") + assert r2.status_code == 302 + assert await _agent_of(db_session, ready) is not None + + +async def test_signup_twice_does_not_create_a_second_agent(client, db_session): + user, r1 = await _signup(client, db_session, "Ada Zephyr", "ada@example.org") + assert r1.status_code == 302 + r2 = await client.post("/agent/request", headers=_auth(user.id)) + assert r2.status_code == 302 and r2.headers["location"] == "/agent" + + rows = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.user_id == user.id) + )).scalars().all() + assert len(rows) == 1 + + # Control: a different user's request *does* add a row, so "still 1" above + # is the dedup guard and not a route that stopped inserting. + other, r3 = await _signup(client, db_session, "Bo Quill", "bo@example.org") + assert r3.status_code == 302 + total = (await db_session.execute(select(AgentRegistry))).scalars().all() + assert len(total) == 2 + assert (await _agent_of(db_session, other)).agent_id == "quill" + + +# =========================================================================== +# 2. The private-channel reopen route +# =========================================================================== + + +async def _reopen(client, world, td, user, guidance="Push on the shared assay."): + return await client.post( + f"/agent/{OWNER_AGENT}/proposals/{td.id}/reopen", + data={"guidance": guidance}, + headers=_auth(user.id), + ) + + +async def test_reopening_the_same_proposal_twice_creates_one_channel( + client, db_session, world, slack +): + """The idempotency guard in reopen_proposal (stale page / Back-button replay). + + Control: a *different* proposal does create a second channel, so "still one" + cannot be satisfied by a reopen that silently stopped working. + """ + r1 = await _reopen(client, world, world.td, world.pi) + assert r1.status_code == 302, r1.text + channels = await _private_channels(db_session) + assert len(channels) == 1 + first_name = channels[0].channel_name + assert first_name.startswith("priv-") + + await db_session.refresh(world.td) + assert world.td.refined_in_channel == channels[0].channel_id + # Both bots + the triggering PI, per specs/privacy-and-channel-visibility.md. + members = (await db_session.execute( + select(PrivateChannelMember).where( + PrivateChannelMember.agent_channel_id == channels[0].id + ) + )).scalars().all() + assert sorted(m.agent_id for m in members if m.agent_id) == sorted( + [OWNER_AGENT, OTHER_AGENT] + ) + assert [m.user_id for m in members if m.user_id] == [world.pi.id] + + # --- the replay ------------------------------------------------------- + r2 = await _reopen(client, world, world.td, world.pi, guidance="Second submit.") + assert r2.status_code == 302 + after = await _private_channels(db_session) + assert len(after) == 1, ( + "the reopen idempotency guard did not hold — the replay minted a second " + f"private channel: {[c.channel_name for c in after]}" + ) + assert after[0].channel_name == first_name + assert len(await _reviews(db_session, OWNER_AGENT)) == 1 + + await db_session.refresh(world.td) + assert world.td.refined_in_channel == after[0].channel_id + + # --- control: a different proposal is not deduped --------------------- + td2 = await factories.make_thread_decision( + db_session, run=world.run, agent_a=OWNER_AGENT, agent_b=OTHER_AGENT, + channel="proteomics", outcome="proposal", summary_text="Summary — A second idea", + ) + await db_session.flush() + r3 = await _reopen(client, world, td2, world.pi, guidance="Different proposal.") + assert r3.status_code == 302 + assert len(await _private_channels(db_session)) == 2 + + assert slack.calls == [], f"the reopen route called Slack: {slack.methods}" + + +async def test_reopen_records_a_rating_zero_review_carrying_the_guidance( + client, db_session, world +): + """The row the idempotency guard keys on. If reopen stopped writing it, the + guard would silently stop working — so pin its shape.""" + assert await _reviews(db_session, OWNER_AGENT) == [] + r = await _reopen(client, world, world.td, world.pi, guidance="Narrow the aims.") + assert r.status_code == 302 + + review = (await _reviews(db_session, OWNER_AGENT))[0] + assert review.rating == 0 + assert review.comment == "[Reopened] Narrow the aims." + assert review.user_id == world.pi.id + assert review.delegate_user_id is None + assert review.submitted_via == "web" + + +async def test_reopen_rejects_empty_guidance(client, db_session, world, slack): + r = await client.post( + f"/agent/{OWNER_AGENT}/proposals/{world.td.id}/reopen", + data={"guidance": " "}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 400 + assert await _private_channels(db_session) == [] + assert slack.calls == [] + + # Control: real guidance on the same proposal does migrate. + assert (await _reopen(client, world, world.td, world.pi)).status_code == 302 + assert len(await _private_channels(db_session)) == 1 + + +async def test_reopen_is_blocked_while_the_agent_is_inactive(client, db_session, world): + world.agent.status = "inactive" + await db_session.flush() + r = await _reopen(client, world, world.td, world.pi) + assert r.status_code == 403 + assert "inactive" in r.json()["detail"].lower() + assert await _private_channels(db_session) == [] + + # Control: reactivating the same agent lets the same request through. + world.agent.status = "active" + await db_session.flush() + assert (await _reopen(client, world, world.td, world.pi)).status_code == 302 + assert len(await _private_channels(db_session)) == 1 + + +async def test_reopen_refuses_a_proposal_the_agent_is_not_part_of( + client, db_session, world +): + foreign = await factories.make_thread_decision( + db_session, run=world.run, agent_a=OTHER_AGENT, agent_b=THIRD_AGENT, + channel="metabolomics", outcome="proposal", + ) + await db_session.flush() + r = await _reopen(client, world, foreign, world.pi) + assert r.status_code == 403 + assert await _private_channels(db_session) == [] + + # Control: the same PI, same route, on a proposal that *is* theirs. + assert (await _reopen(client, world, world.td, world.pi)).status_code == 302 + assert len(await _private_channels(db_session)) == 1 + + +async def test_reopening_an_already_private_thread_reports_not_implemented( + client, db_session, world, slack +): + """The `origin_visibility != 'public'` branch: refuse loudly (501) rather than + fall through to the legacy "post the PI's text in-channel" path.""" + already_private = await factories.make_thread_decision( + db_session, run=world.run, agent_a=OWNER_AGENT, agent_b=OTHER_AGENT, + channel="priv-existing", outcome="proposal", + origin_visibility=VISIBILITY_COLLAB_PRIVATE, + ) + await db_session.flush() + r = await _reopen(client, world, already_private, world.pi) + assert r.status_code == 501 + assert await _private_channels(db_session) == [] + assert await _reviews(db_session, OWNER_AGENT) == [] + assert slack.calls == [] + + # Control: a public-origin proposal on the same route does migrate. + assert (await _reopen(client, world, world.td, world.pi)).status_code == 302 + assert len(await _private_channels(db_session)) == 1 + + +async def test_the_legacy_reopen_finds_a_channel_past_the_first_page( + client, db_session, world, slack, monkeypatch +): + """The legacy (``enable_private_refinement=False``) path resolves the channel + id through the whole of conversations.list, not just page one. + + It used to call ``conversations_list(limit=200)`` once and scan that page, so + a workspace with more channels than fit in one page answered "Channel #x not + found" for a channel that exists — defect 11/12. The route now goes through + ``slack_web.list_channel_ids``, which follows every cursor, so the target on + page **two** below is the whole point of this test. + """ + monkeypatch.setattr(get_settings(), "enable_private_refinement", False) + world.agent.slack_bot_token = "xoxb-fake-for-tests" # flips Slack on + await db_session.flush() + + pages = [ + {"channels": [{"name": "decoy", "id": "C-DECOY"}], + "response_metadata": {"next_cursor": "page2"}}, + {"channels": [{"name": world.td.channel, "id": "C-TARGET"}], + "response_metadata": {"next_cursor": ""}}, + ] + seen: list[dict] = [] + + def _list(**kwargs): + seen.append(kwargs) + return pages[len(seen) - 1] + + slack.stub("conversations_list", _list) + slack.stub("chat_postMessage", {"ok": True, "ts": "1700000000.000900"}) + + assert (await _reopen(client, world, world.td, world.pi)).status_code == 302 + + assert len(seen) == 2, "one page only — the pagination defect is back" + assert seen[1]["cursor"] == "page2" + posted = [kw for name, kw in slack.calls if name == "chat_postMessage"] + assert len(posted) == 1 + assert posted[0]["channel"] == "C-TARGET", ( + "the channel on page two was not resolved" + ) + assert posted[0]["thread_ts"] == world.td.thread_id, ( + "the guidance must stay in the proposal thread, not the channel root" + ) + # Legacy path posts in place: no private refinement channel is minted. + assert await _private_channels(db_session) == [] + + +# =========================================================================== +# 3. Proposal review +# =========================================================================== + + +async def test_a_pi_review_is_recorded_and_cannot_be_submitted_twice( + client, db_session, world +): + url = f"/agent/{OWNER_AGENT}/proposals/{world.td.id}/review" + r = await client.post(url, data={"rating": "3", "comment": " solid "}, + headers=_auth(world.pi.id)) + assert r.status_code == 302 + review = (await _reviews(db_session, OWNER_AGENT))[0] + assert review.rating == 3 + assert review.comment == "solid" + assert review.user_id == world.pi.id + assert review.delegate_user_id is None + assert review.reviewed_by_user_id == world.pi.id + + r2 = await client.post(url, data={"rating": "4"}, headers=_auth(world.pi.id)) + assert r2.status_code == 400 + assert len(await _reviews(db_session, OWNER_AGENT)) == 1 + + # Control: a second proposal is still reviewable, so the 400 is the + # already-reviewed guard rather than a route that broke after one write. + td2 = await factories.make_thread_decision( + db_session, run=world.run, agent_a=OWNER_AGENT, agent_b=OTHER_AGENT, + channel="general", outcome="proposal", + ) + await db_session.flush() + r3 = await client.post( + f"/agent/{OWNER_AGENT}/proposals/{td2.id}/review", + data={"rating": "2"}, headers=_auth(world.pi.id), + ) + assert r3.status_code == 302 + assert len(await _reviews(db_session, OWNER_AGENT)) == 2 + + +@pytest.mark.parametrize("rating", ["0", "5"]) +async def test_review_rejects_out_of_range_ratings(client, db_session, world, rating): + r = await client.post( + f"/agent/{OWNER_AGENT}/proposals/{world.td.id}/review", + data={"rating": rating}, headers=_auth(world.pi.id), + ) + assert r.status_code == 400 + assert await _reviews(db_session, OWNER_AGENT) == [] + + # Control: an in-range rating on the same proposal is accepted. + ok = await client.post( + f"/agent/{OWNER_AGENT}/proposals/{world.td.id}/review", + data={"rating": "1"}, headers=_auth(world.pi.id), + ) + assert ok.status_code == 302 + assert len(await _reviews(db_session, OWNER_AGENT)) == 1 + + +# =========================================================================== +# 4. Delegates: add, act, remove +# =========================================================================== + + +async def test_inviting_a_delegate_creates_a_pending_invitation_and_one_email( + client, db_session, world, sent_emails +): + """One good address and one malformed one in the same submission. + + The malformed address is the absence assertion; the good one is its control. + """ + r = await client.post( + f"/agent/{OWNER_AGENT}/delegates/invite", + data={"emails": "good@example.org, not-an-email"}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + assert "delegate_error" in r.headers["location"] + assert "not-an-email" in unquote(r.headers["location"]) + + rows = (await db_session.execute( + select(DelegateInvitation).where( + DelegateInvitation.agent_registry_id == world.agent.id + ) + )).scalars().all() + assert [x.email for x in rows] == ["good@example.org"] + assert rows[0].status == "pending" + assert rows[0].invited_by_user_id == world.pi.id + assert rows[0].expires_at is not None + + # The email leg is recorded, never sent (plan T11's seam, applied here). + assert len(sent_emails) == 1 + assert sent_emails[0]["to"] == "good@example.org" + assert sent_emails[0]["bot_name"] == "OwnerBot" + assert sent_emails[0]["url"].endswith(f"/invite/{rows[0].token}") + + +async def test_a_duplicate_pending_invitation_is_refused(client, db_session, world): + await _invite(client, world, "dee@example.org") + r = await client.post( + f"/agent/{OWNER_AGENT}/delegates/invite", + data={"emails": "dee@example.org"}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + assert "Invitation+already+pending" in r.headers["location"].replace("%20", "+") + rows = (await db_session.execute( + select(DelegateInvitation).where(DelegateInvitation.email == "dee@example.org") + )).scalars().all() + assert len(rows) == 1 + + # Control: a different address still gets an invitation. + await _invite(client, world, "eve@example.org") + total = (await db_session.execute( + select(DelegateInvitation).where( + DelegateInvitation.agent_registry_id == world.agent.id + ) + )).scalars().all() + assert len(total) == 2 + + +async def test_accepting_an_invitation_creates_the_delegation( + client, db_session, world, delegated +): + """`delegated` builds itself by driving invite → accept; assert the result.""" + assert delegated.row.user_id == delegated.user.id + assert delegated.row.agent_registry_id == world.agent.id + invitation = (await db_session.execute( + select(DelegateInvitation).where(DelegateInvitation.email == "dee@example.org") + )).scalar_one() + assert invitation.status == "accepted" + assert invitation.accepted_by_user_id == delegated.user.id + assert delegated.row.invitation_id == invitation.id + + # And the landing page now routes them to the agent they were added to. + r = await client.get("/agent", headers=_auth(delegated.user.id)) + assert r.status_code == 302 + assert r.headers["location"] == f"/agent/{OWNER_AGENT}/dashboard" + + +async def test_a_delegate_can_review_a_proposal_and_a_stranger_cannot( + client, db_session, world, delegated +): + url = f"/agent/{OWNER_AGENT}/proposals/{world.td.id}/review" + + denied = await client.post(url, data={"rating": "4"}, + headers=_auth(world.stranger.id)) + assert denied.status_code == 403 + assert await _reviews(db_session, OWNER_AGENT) == [] + + allowed = await client.post(url, data={"rating": "4", "comment": "go"}, + headers=_auth(delegated.user.id)) + assert allowed.status_code == 302 + review = (await _reviews(db_session, OWNER_AGENT))[0] + assert review.rating == 4 + # Attribution: the review belongs to the PI, the delegate is recorded + # alongside (specs/web-delegates.md §Changes to ProposalReview). + assert review.user_id == world.pi.id + assert review.delegate_user_id == delegated.user.id + assert review.reviewed_by_user_id == delegated.user.id + + +async def test_removing_a_delegate_revokes_their_access( + client, db_session, world, delegated +): + dash = f"/agent/{OWNER_AGENT}/dashboard" + before = await client.get(dash, headers=_auth(delegated.user.id)) + assert before.status_code == 200, "control failed: the delegate could not read the dashboard" + + r = await client.post( + f"/agent/{OWNER_AGENT}/delegates/{delegated.row.id}/remove", + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + remaining = (await db_session.execute( + select(AgentDelegate).where(AgentDelegate.agent_registry_id == world.agent.id) + )).scalars().all() + assert remaining == [] + + after = await client.get(dash, headers=_auth(delegated.user.id)) + assert after.status_code == 403 + + +async def test_revoking_an_invitation_kills_that_token_only(client, db_session, world): + doomed = await factories.make_user(db_session, name="Dana Doomed", email="doomed@example.org") + keeper = await factories.make_user(db_session, name="Kim Keeper", email="keeper@example.org") + await db_session.flush() + await _invite(client, world, "doomed@example.org") + await _invite(client, world, "keeper@example.org") + doomed_token = await _token_for(db_session, world.agent, "doomed@example.org") + keeper_token = await _token_for(db_session, world.agent, "keeper@example.org") + doomed_inv = (await db_session.execute( + select(DelegateInvitation).where(DelegateInvitation.token == doomed_token) + )).scalar_one() + + r = await client.post( + f"/agent/{OWNER_AGENT}/delegates/{doomed_inv.id}/revoke", + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + statuses = { + row.email: row.status + for row in (await db_session.execute( + select(DelegateInvitation).where( + DelegateInvitation.agent_registry_id == world.agent.id + ) + )).scalars().all() + } + assert statuses == {"doomed@example.org": "revoked", "keeper@example.org": "pending"} + + # The revoked token must not grant access … + dead = await client.post(f"/invite/{doomed_token}/accept", headers=_auth(doomed.id)) + assert dead.status_code == 200 and "no longer valid" in dead.text + # … while the untouched one still does (control). + alive = await client.post(f"/invite/{keeper_token}/accept", headers=_auth(keeper.id)) + assert alive.status_code == 302 + holders = { + d.user_id + for d in (await db_session.execute( + select(AgentDelegate).where(AgentDelegate.agent_registry_id == world.agent.id) + )).scalars().all() + } + assert holders == {keeper.id} + + +async def test_a_delegate_can_link_their_slack_account(client, db_session, world, delegated, slack): + """POST /delegates/connect-slack, with the Slack lookup stubbed.""" + world.agent.slack_bot_token = "xoxb-fake-for-tests" + await db_session.flush() + slack.stub("users_lookupByEmail", {"user": {"id": "U-DELEGATE"}}) + + r = await client.post( + f"/agent/{OWNER_AGENT}/delegates/connect-slack", + headers=_auth(delegated.user.id), + ) + assert r.status_code == 302 and "slack_error" not in r.headers["location"] + assert ("users_lookupByEmail", {"email": "dee@example.org"}) in slack.calls + + agent = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == OWNER_AGENT) + )).scalar_one() + assert agent.delegate_slack_ids == ["U-DELEGATE"] + + +async def test_accepting_an_invitation_syncs_the_delegates_slack_id( + client, db_session, world, slack +): + world.agent.slack_bot_token = "xoxb-fake-for-tests" + await db_session.flush() + slack.stub("users_lookupByEmail", {"user": {"id": "U-DELEGATE"}}) + delegate = await factories.make_user(db_session, name="Dee Legate", email="dee@example.org") + await _invite(client, world, "dee@example.org") + token = await _token_for(db_session, world.agent, "dee@example.org") + + assert (await client.post(f"/invite/{token}/accept", + headers=_auth(delegate.id))).status_code == 302 + agent = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == OWNER_AGENT) + )).scalar_one() + assert agent.delegate_slack_ids == ["U-DELEGATE"] + + +# =========================================================================== +# 5. The remaining read/write routes — enough behaviour to make the auth matrix +# mean something (an endpoint that 403s everyone would satisfy authorization +# tests alone). +# =========================================================================== + + +async def test_the_dashboard_counts_only_this_agents_activity_and_titles_the_proposal( + client, db_session, world +): + """agent_dashboard's three queries and _extract_proposal_title, through the + real template.""" + await factories.make_agent_message( + db_session, run=world.run, agent_id=OWNER_AGENT, phase="new_post" + ) + await factories.make_agent_message( + db_session, run=world.run, agent_id=OWNER_AGENT, phase="thread_reply", + thread_ts="1700000000.000100", + ) + # Control for the agent_id filter: another agent's post must not be counted. + await factories.make_agent_message( + db_session, run=world.run, agent_id=OTHER_AGENT, phase="new_post" + ) + await db_session.flush() + + page = await client.get(f"/agent/{OWNER_AGENT}/dashboard", headers=_auth(world.pi.id)) + assert page.status_code == 200 + assert re.search(r'text-indigo-600">\s*1\s*<', page.text), "posts_count was not 1" + assert re.search(r'text-blue-600">\s*1\s*<', page.text), "threads_count was not 1" + + # The proposal's *title* is the subject, not the ":memo: **Summary — …**" + # boilerplate (the raw summary is still rendered inside the panel). + titles = re.findall(r'text-gray-800 truncate">\s*([^<]*?)\s*<', page.text) + assert titles == ["A shared assay platform"], titles + # Unreviewed → the "agent is paused" banner is shown. + assert "paused from initiating new posts" in page.text + + # Reviewing moves it out of the unreviewed list (the other half). + r = await client.post( + f"/agent/{OWNER_AGENT}/proposals/{world.td.id}/review", + data={"rating": "4"}, headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + page2 = await client.get(f"/agent/{OWNER_AGENT}/dashboard", headers=_auth(world.pi.id)) + assert "paused from initiating new posts" not in page2.text + assert "A shared assay platform" in page2.text + + +async def test_posting_a_message_writes_a_pi_row_into_the_named_channel( + client, db_session, world +): + r = await client.post( + f"/agent/{OWNER_AGENT}/message", + data={"channel_name": "general", "content": " Let's aim at the assay. ", + "tag_bot": "1"}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + assert r.headers["location"] == f"/agent/{OWNER_AGENT}/conversations?posted=1" + + msg = (await db_session.execute( + select(AgentMessage).where(AgentMessage.channel_name == "general") + )).scalar_one() + assert msg.is_bot is False + assert msg.agent_id is None + assert msg.sender_name == "Pat Owner (PI)" + assert msg.content == "@OwnerBot Let's aim at the assay." # tag_bot prepends + assert msg.visibility == "public" + + # …and it is visible on the read view (control that the write is reachable). + page = await client.get(f"/agent/{OWNER_AGENT}/conversations", + headers=_auth(world.pi.id)) + assert page.status_code == 200 + assert "Let's aim at the assay." in page.text or "aim at the assay" in page.text + + +async def test_posting_an_empty_message_is_rejected(client, db_session, world): + r = await client.post( + f"/agent/{OWNER_AGENT}/message", + data={"channel_name": "general", "content": " "}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 400 + assert (await db_session.execute(select(AgentMessage))).scalars().all() == [] + + # Control: non-empty content on the same route does write. + ok = await client.post( + f"/agent/{OWNER_AGENT}/message", + data={"channel_name": "general", "content": "real"}, + headers=_auth(world.pi.id), + ) + assert ok.status_code == 302 + assert len((await db_session.execute(select(AgentMessage))).scalars().all()) == 1 + + +async def test_a_pi_cannot_post_into_another_pairs_private_channel( + client, db_session, world +): + third_user, _ = await _agent_for( + db_session, name="Thea Third", email="thea@example.org", + agent_id=THIRD_AGENT, bot_name="ThirdBot", + ) + foreign_td = await factories.make_thread_decision( + db_session, run=world.run, agent_a=OTHER_AGENT, agent_b=THIRD_AGENT, + channel="metabolomics", outcome="proposal", summary_text="Summary — theirs", + ) + await db_session.flush() + # Produced by the real route, by a PI who is entitled to it. + r = await client.post( + f"/agent/{OTHER_AGENT}/proposals/{foreign_td.id}/reopen", + data={"guidance": "Ours alone."}, + headers=_auth(world.other_pi.id), + ) + assert r.status_code == 302 + private = (await _private_channels(db_session))[0] + assert private.visibility == VISIBILITY_COLLAB_PRIVATE + + await client.post( + f"/agent/{OWNER_AGENT}/message", + data={"channel_name": private.channel_name, "content": "eavesdropping"}, + headers=_auth(world.pi.id), + ) + intruder = (await db_session.execute( + select(AgentMessage).where( + AgentMessage.channel_name == private.channel_name, + AgentMessage.is_bot.is_(False), + ) + )).scalars().all() + assert intruder == [], ( + "a PI with no membership in this collab_private channel wrote into it: " + f"{[m.content for m in intruder]}" + ) + assert third_user is not None + + +async def test_sending_a_dm_records_an_inbound_pi_dm(client, db_session, world): + r = await client.post( + f"/agent/{OWNER_AGENT}/dm", + data={"content": "Always cite the 2019 paper."}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + dm = (await db_session.execute(select(PiDmMessage))).scalar_one() + assert dm.agent_id == OWNER_AGENT + assert dm.direction == "inbound" + assert dm.content == "Always cite the 2019 paper." + assert dm.pi_user_id == f"local:{world.pi.id}" + + +async def test_saving_the_private_profile_persists_to_db_disk_and_a_revision( + client, db_session, world, profiles_dir +): + r = await client.post( + f"/agent/{OWNER_AGENT}/profile/save", + data={"content": "# Private\nUnpublished compound series X."}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 + + profile = (await db_session.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == world.pi.id) + )).scalar_one() + assert "compound series X" in profile.private_profile_md + assert (profiles_dir / "private" / f"{OWNER_AGENT}.md").exists() + revisions = (await db_session.execute( + select(ProfileRevision).where(ProfileRevision.agent_registry_id == world.agent.id) + )).scalars().all() + assert [x.profile_type for x in revisions] == ["private"] + assert revisions[0].changed_by_user_id == world.pi.id + assert revisions[0].mechanism == "web" + + +async def test_saving_the_public_profile_updates_the_pis_profile_not_the_editors( + client, db_session, world, delegated +): + """A delegate edit must land on the PI's ResearcherProfile row.""" + await factories.make_profile(db_session, user=delegated.user, research_summary="Delegate's own") + await db_session.flush() + + r = await client.post( + f"/agent/{OWNER_AGENT}/public-profile/save", + data={ + "research_summary": "Chemical biology of proteostasis.", + "techniques": "cryo-EM, mass spec", + "keywords": "proteostasis", + }, + headers=_auth(delegated.user.id), + ) + assert r.status_code == 302 and "saved=1" in r.headers["location"] + + pi_profile = (await db_session.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == world.pi.id) + )).scalar_one() + assert pi_profile.research_summary == "Chemical biology of proteostasis." + assert pi_profile.techniques == ["cryo-EM", "mass spec"] + + # Control: the delegate's own profile is untouched — a route that wrote to + # current_user's profile would have clobbered this instead. + delegate_profile = (await db_session.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == delegated.user.id) + )).scalar_one() + assert delegate_profile.research_summary == "Delegate's own" + + +async def test_connect_slack_stores_the_pis_slack_user_id(client, db_session, world, slack): + world.agent.slack_bot_token = "xoxb-fake-for-tests" + await db_session.flush() + slack.stub("users_lookupByEmail", {"user": {"id": "U-PI"}}) + + r = await client.post( + f"/agent/{OWNER_AGENT}/slack", + data={"email": "pi@example.org"}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 and "slack_error" not in r.headers["location"] + agent = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == OWNER_AGENT) + )).scalar_one() + assert agent.slack_user_id == "U-PI" + + +async def test_connect_slack_reports_a_lookup_failure_without_writing( + client, db_session, world, slack +): + world.agent.slack_bot_token = "xoxb-fake-for-tests" + await db_session.flush() + + def _not_found(**kwargs): + raise RuntimeError("users_not_found") + + slack.stub("users_lookupByEmail", _not_found) + r = await client.post( + f"/agent/{OWNER_AGENT}/slack", + data={"email": "nobody@example.org"}, + headers=_auth(world.pi.id), + ) + assert r.status_code == 302 and "slack_error" in r.headers["location"] + agent = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == OWNER_AGENT) + )).scalar_one() + assert agent.slack_user_id is None + + # Control: the same route with a resolving lookup does write. + slack.stub("users_lookupByEmail", {"user": {"id": "U-PI"}}) + assert (await client.post( + f"/agent/{OWNER_AGENT}/slack", data={"email": "pi@example.org"}, + headers=_auth(world.pi.id), + )).status_code == 302 + agent = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == OWNER_AGENT) + )).scalar_one() + assert agent.slack_user_id == "U-PI" + + +# =========================================================================== +# 6. Authorization, all 19 endpoints +# =========================================================================== + + +@dataclass(frozen=True) +class Ep: + method: str + route: str # exactly as registered, for the inventory check + template: str # concrete path, .format(**ctx) + data: dict = field(default_factory=dict) + agent_scoped: bool = True # goes through get_agent_with_access + owner_only: bool = False # rejects delegates (specs/web-delegates.md) + + @property + def id(self) -> str: + return f"{self.method} {self.route}" + + +ENDPOINTS: list[Ep] = [ + Ep("GET", "/agent", "/agent", agent_scoped=False), + Ep("POST", "/agent/request", "/agent/request", agent_scoped=False), + Ep("GET", "/agent/{agent_id}/dashboard", "/agent/{agent}/dashboard"), + Ep("GET", "/agent/{agent_id}/conversations", "/agent/{agent}/conversations"), + Ep("GET", "/agent/{agent_id}/thread/{message_ts}", "/agent/{agent}/thread/{ts}"), + Ep("POST", "/agent/{agent_id}/message", "/agent/{agent}/message", + {"channel_name": "general", "content": "hello"}), + Ep("POST", "/agent/{agent_id}/dm", "/agent/{agent}/dm", {"content": "directive"}), + Ep("GET", "/agent/{agent_id}/profile", "/agent/{agent}/profile"), + Ep("GET", "/agent/{agent_id}/profile/edit", "/agent/{agent}/profile/edit"), + Ep("POST", "/agent/{agent_id}/profile/save", "/agent/{agent}/profile/save", + {"content": "# Private"}), + Ep("GET", "/agent/{agent_id}/public-profile", "/agent/{agent}/public-profile"), + Ep("GET", "/agent/{agent_id}/public-profile/edit", "/agent/{agent}/public-profile/edit"), + Ep("POST", "/agent/{agent_id}/public-profile/save", "/agent/{agent}/public-profile/save", + {"research_summary": "s", "techniques": "a,b", "keywords": "k"}), + Ep("POST", "/agent/{agent_id}/proposals/{thread_decision_id}/review", + "/agent/{agent}/proposals/{td}/review", {"rating": "3"}), + Ep("POST", "/agent/{agent_id}/proposals/{thread_decision_id}/reopen", + "/agent/{agent}/proposals/{td}/reopen", {"guidance": "refine the aims"}), + Ep("POST", "/agent/{agent_id}/slack", "/agent/{agent}/slack", + {"email": "pi@example.org"}, owner_only=True), + Ep("POST", "/agent/{agent_id}/delegates/connect-slack", + "/agent/{agent}/delegates/connect-slack"), + Ep("POST", "/agent/{agent_id}/delegates/invite", "/agent/{agent}/delegates/invite", + {"emails": "fresh@example.org"}, owner_only=True), + Ep("POST", "/agent/{agent_id}/delegates/{invitation_id}/revoke", + "/agent/{agent}/delegates/{inv}/revoke", owner_only=True), + Ep("POST", "/agent/{agent_id}/delegates/{delegate_id}/remove", + "/agent/{agent}/delegates/{dele}/remove", owner_only=True), +] + +AGENT_SCOPED = [e for e in ENDPOINTS if e.agent_scoped] + + +def test_the_endpoint_table_matches_the_registered_routes(): + """A new endpoint on agent_page.py must show up here as a missing entry. + + Without this the parametrised authorization tests silently keep passing + while an unprotected route ships. + """ + from src.routers import agent_page + + registered = { + (method, "/agent" + route.path) + for route in agent_page.router.routes + for method in route.methods + if method not in ("HEAD", "OPTIONS") + } + listed = {(e.method, e.route) for e in ENDPOINTS} + assert registered == listed, ( + f"missing from ENDPOINTS: {sorted(registered - listed)}; " + f"stale entries: {sorted(listed - registered)}" + ) + assert len(ENDPOINTS) == 20 + + +def _path(ep: Ep, world, delegated=None, ts: str = "0.0000") -> str: + return ep.template.format( + agent=OWNER_AGENT, + td=world.td.id, + inv=delegated.pending_invitation.id if delegated else uuid.uuid4(), + dele=delegated.row.id if delegated else uuid.uuid4(), + ts=ts, + ) + + +@pytest.fixture +async def thread_root(db_session, world) -> str: + """A real thread ROOT belonging to OWNER_AGENT, for the + /agent/{agent_id}/thread/{message_ts} authorization tests. + + Not folded into `world` itself: several tests assert an exact, + unfiltered count of `AgentMessage` rows (e.g. + test_posting_an_empty_message_is_rejected), so a message seeded into the + shared fixture would silently change what they are counting. Without a + ts that actually resolves, the owner's positive-control request would 404 + — which is not in the {200, 302} the authorization tests accept — and + would mask a stranger's 403 test passing for the wrong reason. + """ + msg = await factories.make_agent_message( + db_session, run=world.run, agent_id=OWNER_AGENT, channel_name="general", + channel_id="C-THREAD-ROOT", phase="new_post", message_ts="50.0001", + sender_name="OwnerBot", content="root", visibility="public", + ) + await db_session.flush() + return msg.message_ts + + +@pytest.mark.parametrize("ep", ENDPOINTS, ids=[e.id for e in ENDPOINTS]) +async def test_every_endpoint_redirects_a_logged_out_visitor(client, world, delegated, ep): + # The `delegated` fixture authenticated as the PI and the delegate on this + # same httpx client; empty the jar so "logged out" means exactly that and + # cannot be satisfied (or broken) by a leftover Set-Cookie. + client.cookies.clear() + r = await client.request(ep.method, _path(ep, world, delegated), data=ep.data) + assert r.status_code == 302, f"{ep.id} returned {r.status_code} to an anonymous caller" + assert r.headers["location"].startswith("/login"), r.headers["location"] + + +@pytest.mark.parametrize("ep", AGENT_SCOPED, ids=[e.id for e in AGENT_SCOPED]) +async def test_a_stranger_cannot_touch_an_agent_they_do_not_own( + client, world, delegated, ep, slack, thread_root +): + """The half worth most: a logged-in user with no relationship to the agent. + + Positive control in the same test — the *owner* making the identical request + is not rejected, so a route that 403s everyone (or one that 404s because the + fixture URL is wrong) cannot pass this. + """ + slack.stub("users_lookupByEmail", {"user": {"id": "U-PI"}}) + path = _path(ep, world, delegated, ts=thread_root) + + denied = await client.request(ep.method, path, data=ep.data, + headers=_auth(world.stranger.id)) + assert denied.status_code == 403, ( + f"{ep.id} let a stranger through with {denied.status_code}" + ) + assert denied.json()["detail"] == "Access denied" + + allowed = await client.request(ep.method, path, data=ep.data, + headers=_auth(world.pi.id)) + assert allowed.status_code in (200, 302), f"{ep.id} owner control got {allowed.status_code}" + if allowed.status_code == 302: + assert not allowed.headers["location"].startswith("/login") + + +@pytest.mark.parametrize("ep", AGENT_SCOPED, ids=[e.id for e in AGENT_SCOPED]) +async def test_delegate_write_access_matches_the_spec( + client, world, delegated, ep, slack, thread_root +): + """Delegates get everything except delegate management and Slack linking of + the PI's own account (specs/web-delegates.md §Write access differentiation). + + Both halves are in this one parametrisation: the owner-only endpoints must + reject, and every other endpoint must accept. + """ + slack.stub("users_lookupByEmail", {"user": {"id": "U-DELEGATE"}}) + r = await client.request(ep.method, _path(ep, world, delegated, ts=thread_root), + data=ep.data, headers=_auth(delegated.user.id)) + if ep.owner_only: + assert r.status_code == 403, f"{ep.id} should be PI-only, got {r.status_code}" + assert "Only the PI" in r.json()["detail"] + else: + assert r.status_code in (200, 302), f"{ep.id} refused a delegate ({r.status_code})" + if r.status_code == 302: + assert not r.headers["location"].startswith("/login") + + +async def test_an_unknown_agent_id_is_a_404_not_a_403(client, world): + """Distinguishes "no such agent" from "not yours" — a 403 here would leak + nothing, but a 200 would mean the lookup was skipped entirely.""" + r = await client.get("/agent/nosuchagent/dashboard", headers=_auth(world.pi.id)) + assert r.status_code == 404 + # Control: the real slug is reachable for the same user. + ok = await client.get(f"/agent/{OWNER_AGENT}/dashboard", headers=_auth(world.pi.id)) + assert ok.status_code == 200 + + +async def test_a_pending_agent_cannot_reach_the_dashboard(client, db_session, world): + world.agent.status = "pending" + await db_session.flush() + r = await client.get(f"/agent/{OWNER_AGENT}/dashboard", headers=_auth(world.pi.id)) + assert r.status_code == 302 and r.headers["location"] == "/agent" + + # Control: active reaches it. (inactive is allowed in too — the dashboard + # gates the reopen action separately, see agent_dashboard's docstring.) + world.agent.status = "inactive" + await db_session.flush() + assert (await client.get(f"/agent/{OWNER_AGENT}/dashboard", + headers=_auth(world.pi.id))).status_code == 200 diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py new file mode 100644 index 0000000..6261278 --- /dev/null +++ b/tests/integration/test_cli.py @@ -0,0 +1,762 @@ +"""Integration tests for the management CLI (src/cli.py) — Task 6 of the full-system plan. + +Seven commands, previously zero coverage. `seed-profiles` is the documented way to add +PIs (CLAUDE.md), so it is a production path. + +**Why these tests are shaped oddly.** The CLI does not take a session; every command +calls `src.cli._get_db()`, which builds its *own* engine from +`get_settings().database_url` and commits for real. Three consequences: + +1. The conftest `db_session` (rolled-back transaction) is invisible to the CLI and + useless here — the CLI reads on a different connection. These tests therefore use + the session-scoped `engine` with a *committing* session and clean up explicitly. +2. Every test function is synchronous. `_run()` is `asyncio.run()`, which raises if a + loop is already running, so a pytest-asyncio coroutine test could not invoke a + command at all. +3. Inside the app container `DATABASE_URL` points at the shared dev database, which the + plan puts off-limits. `cli_points_at_test_db` (autouse) repoints `get_settings` at + the migrated test DB and refuses to run if that failed. + +The only mocked dependency is `src.services.orcid.fetch_orcid_profile` — the CLI's sole +outbound call. No LLM is reachable from these commands: they enqueue `Job` rows and the +worker does the generating. +""" + +import asyncio +import uuid + +import pytest +from sqlalchemy import delete, func, select, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from typer.testing import CliRunner + +from src.cli import app as cli_app +from src.models import AgentRegistry, Job, ProfileRevision, User +from tests import factories + +pytestmark = pytest.mark.integration + +# Rows created by this module are committed for real, so they are tagged and deleted +# around every test. ORCID is a free-form String(50); nothing validates the format. +ORCID_PREFIX = "CLI-TEST-" +AGENT_PREFIX = "clitest" + + +def _orcid(tag: str) -> str: + return f"{ORCID_PREFIX}{tag}" + + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + + +def _in_own_loop(engine, fn): + """Run `fn(session)` in a fresh event loop on a committing session. + + A fresh loop per call is required: the CLI's own `asyncio.run()` closes the loop it + made, and the conftest engine uses NullPool precisely so a new loop gets a new + asyncpg connection. + """ + + async def _inner(): + factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with factory() as session: + out = await fn(session) + await session.commit() + return out + + return asyncio.run(_inner()) + + +@pytest.fixture +def db(engine): + """Call as `db(lambda s: some_coroutine(s))`; commits, returns the value.""" + + def _call(fn): + return _in_own_loop(engine, fn) + + return _call + + +async def _wipe(session): + agent_ids = select(AgentRegistry.id).where(AgentRegistry.agent_id.like(f"{AGENT_PREFIX}%")) + user_ids = select(User.id).where(User.orcid.like(f"{ORCID_PREFIX}%")) + await session.execute( + delete(ProfileRevision).where(ProfileRevision.agent_registry_id.in_(agent_ids)) + ) + await session.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.like(f"{AGENT_PREFIX}%"))) + await session.execute(delete(Job).where(Job.user_id.in_(user_ids))) + await session.execute(delete(User).where(User.orcid.like(f"{ORCID_PREFIX}%"))) + + +@pytest.fixture(autouse=True) +def _clean_slate(engine): + """Delete this module's rows before and after each test (the CLI really commits).""" + _in_own_loop(engine, _wipe) + yield + _in_own_loop(engine, _wipe) + + +@pytest.fixture(autouse=True) +def cli_points_at_test_db(monkeypatch, pg_url): + """Repoint `_get_db()` at the migrated test database. + + `_get_db` does `from src.config import get_settings` *inside* the function, so + patching the module attribute is enough — the real `_get_db` still runs. + """ + from src import config + + patched = config.get_settings().model_copy(update={"database_url": pg_url}) + monkeypatch.setattr(config, "get_settings", lambda: patched) + monkeypatch.setenv("DATABASE_URL", pg_url) + + # Guard, not decoration: in the app container the ambient DATABASE_URL is the + # shared dev DB. Writing there would be a plan violation, so refuse to proceed. + assert patched.database_url == pg_url + assert not pg_url.rstrip("/").endswith("/copi"), f"refusing to run against {pg_url}" + return patched + + +@pytest.fixture +def runner(): + # COLUMNS keeps rich from wrapping the list-users table mid-cell at its 80-column + # non-tty default, which would break substring assertions for reasons unrelated to + # the code under test. + return CliRunner(env={"COLUMNS": "220", "TERM": "dumb", "NO_COLOR": "1"}) + + +class _OrcidStub: + """Recording stand-in for src.services.orcid.fetch_orcid_profile.""" + + def __init__(self): + self.calls: list[str] = [] + self.profiles: dict[str, dict] = {} + self.fail: set[str] = set() + + def set(self, orcid: str, **fields): + self.profiles[orcid] = {"orcid": orcid, **fields} + + async def __call__(self, orcid: str): + self.calls.append(orcid) + if orcid in self.fail: + raise RuntimeError("ORCID 503") + return self.profiles.get(orcid, {"orcid": orcid, "name": f"Stub {orcid}"}) + + +@pytest.fixture +def orcid_stub(monkeypatch): + stub = _OrcidStub() + monkeypatch.setattr("src.services.orcid.fetch_orcid_profile", stub) + return stub + + +def _ok(result): + assert result.exit_code == 0, ( + f"exit={result.exit_code} exception={result.exception!r}\n{result.output}" + ) + return result + + +# --------------------------------------------------------------------------- +# Queries +# --------------------------------------------------------------------------- + + +async def _user_by_orcid(session, orcid): + return (await session.execute(select(User).where(User.orcid == orcid))).scalar_one_or_none() + + +async def _count_users(session, orcid): + return ( + await session.execute(select(func.count()).select_from(User).where(User.orcid == orcid)) + ).scalar_one() + + +async def _mine(session): + """Every user this module created, oldest first.""" + rows = await session.execute( + select(User).where(User.orcid.like(f"{ORCID_PREFIX}%")).order_by(User.created_at) + ) + return list(rows.scalars()) + + +async def _jobs_for(session, user_id): + rows = await session.execute( + select(Job).where(Job.user_id == user_id).order_by(Job.enqueued_at) + ) + return list(rows.scalars()) + + +async def _all_job_ids(session): + return {row[0] for row in await session.execute(select(Job.id))} + + +async def _all_user_ids(session): + return {row[0] for row in await session.execute(select(User.id))} + + +async def _jobs_by_id(session, ids): + if not ids: + return [] + rows = await session.execute(select(Job).where(Job.id.in_(list(ids)))) + return list(rows.scalars()) + + +async def _revisions_for(session, agent_registry_id): + rows = await session.execute( + select(ProfileRevision).where(ProfileRevision.agent_registry_id == agent_registry_id) + ) + return list(rows.scalars()) + + +# =========================================================================== +# T6.1 — seed-profile / seed-profiles +# =========================================================================== + + +def test_seed_profile_creates_user_and_enqueues_job(db, runner, orcid_stub): + """T6.1: the happy path writes the ORCID payload through to the row and the job.""" + orcid = _orcid("seed1") + orcid_stub.set( + orcid, + name="Ada Lovelace", + email="ada@example.edu", + institution="Analytical Institute", + department="Engines", + ) + + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", orcid])) + + user = db(lambda s: _user_by_orcid(s, orcid)) + assert user is not None, "seed-profile exited 0 but created no user" + # Every field the command claims to copy across, so dropping one is caught. + assert user.name == "Ada Lovelace" + assert user.email == "ada@example.edu" + assert user.institution == "Analytical Institute" + assert user.department == "Engines" + assert orcid_stub.calls == [orcid] + + jobs = db(lambda s: _jobs_for(s, user.id)) + assert len(jobs) == 1, f"expected exactly one job, got {len(jobs)}" + assert jobs[0].type == "generate_profile" + assert jobs[0].status == "pending" + assert jobs[0].payload == {"user_id": str(user.id), "orcid": orcid} + + +def test_seed_profile_duplicate_orcid_does_not_create_a_second_user(db, runner, orcid_stub): + """T6.1 control: re-seeding an ORCID is a no-op for `users`; a *new* ORCID is not.""" + dup = _orcid("dup") + fresh = _orcid("fresh") + orcid_stub.set(dup, name="First Seed") + orcid_stub.set(fresh, name="Second Seed") + + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", dup])) + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", dup])) + + assert db(lambda s: _count_users(s, dup)) == 1 + # ORCID was fetched once only — the existence check short-circuits the network call. + assert orcid_stub.calls == [dup] + + # Positive control: the guard is not "never create anything". + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", fresh])) + assert db(lambda s: _count_users(s, fresh)) == 1 + assert len(db(_mine)) == 2 + assert orcid_stub.calls == [dup, fresh] + + # Characterization: the *job* is re-enqueued on every run even for an existing + # user. That is how the command doubles as "regenerate this one PI". + user = db(lambda s: _user_by_orcid(s, dup)) + assert len(db(lambda s: _jobs_for(s, user.id))) == 2 + + +def test_seed_profile_no_pipeline_skips_the_job_but_still_creates_the_user( + db, runner, orcid_stub +): + """T6.1: --no-pipeline suppresses the job. Control: without it, a job appears.""" + quiet = _orcid("nopipe") + loud = _orcid("pipe") + orcid_stub.set(quiet, name="Quiet PI") + orcid_stub.set(loud, name="Loud PI") + + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", quiet, "--no-pipeline"])) + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", loud])) + + quiet_user = db(lambda s: _user_by_orcid(s, quiet)) + loud_user = db(lambda s: _user_by_orcid(s, loud)) + assert quiet_user is not None and quiet_user.name == "Quiet PI" + assert db(lambda s: _jobs_for(s, quiet_user.id)) == [] + # Control for the absence assertion above. + assert len(db(lambda s: _jobs_for(s, loud_user.id))) == 1 + + +def test_seed_profile_falls_back_to_the_bare_orcid_when_the_lookup_fails( + db, runner, orcid_stub +): + """T6.1: an ORCID outage still yields a user row (name == the ORCID itself). + + Control in the same test: a working lookup in the same run keeps the real name, so + "name == orcid" cannot be what the command always does. + """ + broken = _orcid("broken") + working = _orcid("working") + orcid_stub.fail.add(broken) + orcid_stub.set(working, name="Reachable PI", institution="Somewhere") + + result = _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", broken])) + assert "Failed to fetch ORCID profile" in result.output + + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", working])) + + fallback = db(lambda s: _user_by_orcid(s, broken)) + assert fallback is not None, "an ORCID outage must not lose the user entirely" + assert fallback.name == broken + assert fallback.institution is None + assert len(db(lambda s: _jobs_for(s, fallback.id))) == 1 + + good = db(lambda s: _user_by_orcid(s, working)) + assert good.name == "Reachable PI" and good.institution == "Somewhere" + + +def test_seed_profiles_reads_the_file_and_ignores_comments_and_blanks( + db, runner, orcid_stub, tmp_path +): + """T6.1: the documented bulk path (CLAUDE.md puts `# comment` lines in orcids.txt). + + Control for "comments create nothing": the two real lines in the same file do. + """ + a, b = _orcid("file-a"), _orcid("file-b") + orcid_stub.set(a, name="File PI A") + orcid_stub.set(b, name="File PI B") + listing = tmp_path / "orcids.txt" + listing.write_text(f"# Cohort 3\n\n \n{a}\n {b} \n") + + _ok(runner.invoke(cli_app, ["seed-profiles", "--file", str(listing)])) + + created = {u.orcid: u for u in db(_mine)} + assert set(created) == {a, b}, f"unexpected user set {sorted(created)}" + assert created[a].name == "File PI A" and created[b].name == "File PI B" + # Whitespace is stripped before the lookup, not passed through as part of the id. + assert orcid_stub.calls == [a, b] + for user in created.values(): + assert len(db(lambda s, uid=user.id: _jobs_for(s, uid))) == 1 + + +def test_seed_profiles_missing_file_exits_nonzero_and_writes_nothing( + db, runner, orcid_stub, tmp_path +): + """T6.1: a bad --file is a loud failure. Control: a good --file in the same test.""" + missing = tmp_path / "not-here.txt" + result = runner.invoke(cli_app, ["seed-profiles", "--file", str(missing)]) + assert result.exit_code == 1, f"missing file must exit nonzero, got {result.exit_code}" + assert "File not found" in result.output + assert db(_mine) == [] + assert orcid_stub.calls == [] + + good = tmp_path / "good.txt" + real = _orcid("file-ok") + orcid_stub.set(real, name="Present PI") + good.write_text(real + "\n") + _ok(runner.invoke(cli_app, ["seed-profiles", "--file", str(good)])) + assert [u.orcid for u in db(_mine)] == [real] + + +# =========================================================================== +# T6.2 — admin:grant / admin:revoke +# =========================================================================== + + +def test_admin_grant_and_revoke_flip_is_admin_and_are_idempotent(db, runner): + """T6.2: both directions, each run twice, plus a bystander that must not move.""" + target_orcid = _orcid("admin-target") + bystander_orcid = _orcid("admin-bystander") + + async def _seed(session): + await factories.make_user(session, orcid=target_orcid, name="Target PI", is_admin=False) + await factories.make_user( + session, orcid=bystander_orcid, name="Bystander PI", is_admin=False + ) + + db(_seed) + + def _is_admin(orcid): + return db(lambda s: _user_by_orcid(s, orcid)).is_admin + + result = _ok(runner.invoke(cli_app, ["admin:grant", "--orcid", target_orcid])) + assert "Granted admin to Target PI" in result.output + assert _is_admin(target_orcid) is True + # Idempotent: a second grant leaves it granted (and does not error). + _ok(runner.invoke(cli_app, ["admin:grant", "--orcid", target_orcid])) + assert _is_admin(target_orcid) is True + # Scoped: the update is keyed on the ORCID, not applied to the table. + assert _is_admin(bystander_orcid) is False + + _ok(runner.invoke(cli_app, ["admin:revoke", "--orcid", target_orcid])) + assert _is_admin(target_orcid) is False + _ok(runner.invoke(cli_app, ["admin:revoke", "--orcid", target_orcid])) + assert _is_admin(target_orcid) is False + + # Control for revoke's scope: grant the bystander, revoke the target, and check + # only the target moved. + _ok(runner.invoke(cli_app, ["admin:grant", "--orcid", bystander_orcid])) + _ok(runner.invoke(cli_app, ["admin:revoke", "--orcid", target_orcid])) + assert _is_admin(bystander_orcid) is True + assert _is_admin(target_orcid) is False + + +def test_admin_grant_on_unknown_orcid_changes_nothing_and_says_so(db, runner): + """T6.2 control: an unknown ORCID must not silently create or promote anybody. + + (The plan says "email"; the command's flag is --orcid, which is the lookup key.) + """ + real_orcid = _orcid("admin-real") + ghost = _orcid("admin-ghost") + + async def _seed(session): + await factories.make_user(session, orcid=real_orcid, name="Real PI", is_admin=False) + + db(_seed) + before = len(db(_mine)) + + result = runner.invoke(cli_app, ["admin:grant", "--orcid", ghost]) + assert f"User with ORCID {ghost} not found" in result.output + assert db(lambda s: _user_by_orcid(s, ghost)) is None, "grant must not create users" + assert len(db(_mine)) == before + assert db(lambda s: _user_by_orcid(s, real_orcid)).is_admin is False + + revoke = runner.invoke(cli_app, ["admin:revoke", "--orcid", ghost]) + assert f"User with ORCID {ghost} not found" in revoke.output + assert db(lambda s: _user_by_orcid(s, ghost)) is None + + # Positive control: the same invocation against a real ORCID does work, so the + # "nothing happened" assertions above are not vacuous. + _ok(runner.invoke(cli_app, ["admin:grant", "--orcid", real_orcid])) + assert db(lambda s: _user_by_orcid(s, real_orcid)).is_admin is True + + +def test_admin_grant_on_unknown_orcid_should_exit_nonzero(runner): + result = runner.invoke(cli_app, ["admin:grant", "--orcid", _orcid("admin-nobody")]) + assert result.exit_code != 0 + + +# =========================================================================== +# T6.3 — list-users +# =========================================================================== + + +def test_list_users_renders_real_rows_including_the_null_institution_case(db, runner): + """T6.3: smoke test over real data shapes, with per-row flags checked, not just + "the command ran". Two users differing in every rendered flag pin the columns.""" + admin_orcid = _orcid("list-admin") + plain_orcid = _orcid("list-plain") + + async def _seed(session): + await factories.make_user( + session, + orcid=admin_orcid, + name="Listed Admin", + institution="Scripps Research", + is_admin=True, + onboarding_complete=True, + ) + # institution=None exercises the "—" fallback branch. + await factories.make_user( + session, + orcid=plain_orcid, + name="Listed Plain", + institution=None, + is_admin=False, + onboarding_complete=False, + ) + + db(_seed) + + result = _ok(runner.invoke(cli_app, ["list-users"])) + out = result.output + assert "Users" in out + assert "Listed Admin" in out and "Listed Plain" in out + assert admin_orcid in out and plain_orcid in out + + def _row(orcid): + matches = [ln for ln in out.splitlines() if orcid in ln] + assert len(matches) == 1, f"expected one rendered row for {orcid}, got {matches}" + return matches[0] + + admin_row = _row(admin_orcid) + plain_row = _row(plain_orcid) + assert "Scripps Research" in admin_row + assert "—" in plain_row, "a null institution should render the em-dash placeholder" + # Admin + Onboarded are the last two cells. The admin row is Yes/Yes and the plain + # row No/No, so a hard-coded flag column cannot satisfy both. + assert (admin_row.count("Yes"), admin_row.count("No")) == (2, 0), admin_row + assert (plain_row.count("Yes"), plain_row.count("No")) == (0, 2), plain_row + + +# =========================================================================== +# T6.4 — regenerate-profiles +# =========================================================================== + + +def test_regenerate_profiles_enqueues_exactly_one_job_per_eligible_user(db, runner): + """T6.4 (the enqueue half): one new job per user, correct payload, none doubled.""" + orcids = [_orcid(f"regen-{i}") for i in range(3)] + + async def _seed(session): + for i, orcid in enumerate(orcids): + await factories.make_user(session, orcid=orcid, name=f"Regen PI {i}") + + db(_seed) + + before_jobs = db(_all_job_ids) + all_users = db(_all_user_ids) + + result = _ok(runner.invoke(cli_app, ["regenerate-profiles"])) + assert f"Enqueued {len(all_users)} profile regeneration jobs." in result.output + + after_jobs = db(_all_job_ids) + new_ids = after_jobs - before_jobs + new_jobs = db(lambda s: _jobs_by_id(s, new_ids)) + + # Exactly one job per user in the table — catches both "skipped someone" and + # "enqueued twice", which a bare count of 3 would not. + assert {j.user_id for j in new_jobs} == all_users + assert len(new_jobs) == len(all_users) + + by_user = {j.user_id: j for j in new_jobs} + for user in db(_mine): + job = by_user[user.id] + assert job.type == "generate_profile" + assert job.status == "pending" + assert job.payload == {"user_id": str(user.id), "orcid": user.orcid} + + +def test_regenerate_profiles_ineligible_set_is_empty_because_orcid_is_not_null(db, runner): + """T6.4 (the skip half), honestly. + + The command filters `User.orcid.isnot(None)`, but `users.orcid` is NOT NULL in both + the model and migration 0001/0010 — so the ineligible class cannot be populated and + the filter is unreachable. Pin that fact instead of faking a row the schema forbids; + if someone makes the column nullable, this test goes red and real skip coverage + becomes writable. + """ + assert User.__table__.c.orcid.nullable is False + + async def _nullability(session, column): + return ( + await session.execute( + text( + "SELECT is_nullable FROM information_schema.columns " + "WHERE table_name = 'users' AND column_name = :col" + ), + {"col": column}, + ) + ).scalar_one() + + assert db(lambda s: _nullability(s, "orcid")) == "NO" + # Control: the same query reports YES for a genuinely nullable column, so "NO" is + # a real answer and not an artefact of the query. + assert db(lambda s: _nullability(s, "institution")) == "YES" + + # And the eligible half still fires, so the command is not simply doing nothing. + orcid = _orcid("regen-eligible") + + async def _seed(session): + await factories.make_user(session, orcid=orcid, name="Eligible PI") + + db(_seed) + before = db(_all_job_ids) + _ok(runner.invoke(cli_app, ["regenerate-profiles"])) + user = db(lambda s: _user_by_orcid(s, orcid)) + new_ids = db(_all_job_ids) - before + new_jobs = db(lambda s: _jobs_by_id(s, new_ids)) + assert [j.user_id for j in new_jobs].count(user.id) == 1 + + +# =========================================================================== +# T6.5 — backfill-profile-revisions +# =========================================================================== + + +def _write_profiles(tmp_path, files: dict[str, str]): + """files maps 'public/alpha.md' -> content, under tmp_path/profiles/.""" + for rel, content in files.items(): + path = tmp_path / "profiles" / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +@pytest.fixture +def backfill_fixture(db, monkeypatch, tmp_path): + """Registered agent `alpha` with three profile files, plus two files that must be + skipped: one for an unregistered agent and one that is empty. + + The command resolves `profiles/` relative to the process CWD, so the test + chdirs into a temp tree; otherwise it would read (and backfill from) /app/profiles. + """ + alpha_id = f"{AGENT_PREFIX}alpha" + beta_id = f"{AGENT_PREFIX}beta" + + async def _seed(session): + alpha = await factories.make_agent(session, agent_id=alpha_id, bot_name="ClitestAlphaBot") + beta = await factories.make_agent(session, agent_id=beta_id, bot_name="ClitestBetaBot") + return alpha.id, beta.id + + alpha_uuid, beta_uuid = db(_seed) + + _write_profiles( + tmp_path, + { + f"public/{alpha_id}.md": "# Alpha public\nPeptides.\n", + f"private/{alpha_id}.md": "# Alpha private\nUnpublished.\n", + f"memory/{alpha_id}.md": "# Alpha memory\nNotes.\n", + f"public/{beta_id}.md": " \n", # empty-after-strip: skipped + f"public/{AGENT_PREFIX}ghost.md": "# Ghost\nNo registry row.\n", + }, + ) + monkeypatch.chdir(tmp_path) + return { + "alpha_id": alpha_id, + "beta_id": beta_id, + "alpha_uuid": alpha_uuid, + "beta_uuid": beta_uuid, + "tmp_path": tmp_path, + } + + +def test_backfill_creates_one_revision_per_profile_file_and_skips_the_rest( + db, runner, backfill_fixture +): + """T6.5 (first run): three revisions for the registered agent with non-empty files. + + Both absence assertions have their control in this same run — the ghost file and + the empty file produce nothing while alpha's three files produce three rows. + """ + fx = backfill_fixture + result = _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + + assert "Created 3 profile revisions." in result.output + assert f"no agent '{AGENT_PREFIX}ghost'" in result.output + + revisions = db(lambda s: _revisions_for(s, fx["alpha_uuid"])) + assert len(revisions) == 3 + by_type = {r.profile_type: r for r in revisions} + assert set(by_type) == {"public", "private", "memory"} + for profile_type, revision in by_type.items(): + expected = (fx["tmp_path"] / "profiles" / profile_type / f"{fx['alpha_id']}.md").read_text() + assert revision.content == expected + assert revision.mechanism == "pipeline" + assert revision.change_summary == "Initial backfill from existing file" + assert revision.changed_by_user_id is None + + # Whitespace-only file: registered agent, still no revision (control = the 3 above). + assert db(lambda s: _revisions_for(s, fx["beta_uuid"])) == [] + + +def test_backfill_run_twice_does_not_duplicate_any_revision(db, runner, backfill_fixture): + """Regression for the T6.5 bug, with the evidence in one place. + + `create_revision` (src/services/profile_versioning.py) used to append + unconditionally and the command never checked for an existing row, so a second + backfill wrote a second byte-identical revision for every file — the duplicates + being identical is what made them useless as history. The second run must now + report creating nothing and leave the row count alone. + """ + fx = backfill_fixture + + first = _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + assert "Created 3 profile revisions." in first.output + # Control: the first run really did create rows, so "unchanged" would mean something. + assert len(db(lambda s: _revisions_for(s, fx["alpha_uuid"]))) == 3 + + second = _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + assert "Created 0 profile revisions." in second.output + assert f"Unchanged public profile for {fx['alpha_id']}" in second.output + revisions = db(lambda s: _revisions_for(s, fx["alpha_uuid"])) + assert len(revisions) == 3, "a re-run must not duplicate anything" + # One revision per (type, content) pair — no identical siblings. + assert len({(r.profile_type, r.content) for r in revisions}) == 3 + + +def test_backfill_is_idempotent(db, runner, backfill_fixture): + fx = backfill_fixture + _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + after_first = len(db(lambda s: _revisions_for(s, fx["alpha_uuid"]))) + assert after_first == 3 + _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + assert len(db(lambda s: _revisions_for(s, fx["alpha_uuid"]))) == after_first + + +def test_a_changed_profile_body_still_creates_a_new_revision(db, runner, backfill_fixture): + """The other half of the idempotency guard, and the reason it keys on content. + + `create_revision` skips a write only when the newest revision for that + (agent, profile_type) is byte-identical. A guard that also swallowed real edits + would be a worse bug than the duplication it replaced — it would silently drop + history — so this pins the positive case: edit one file, re-run, get one more + revision for that type and none for the two untouched ones. + """ + fx = backfill_fixture + _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + assert len(db(lambda s: _revisions_for(s, fx["alpha_uuid"]))) == 3 + + edited = "# Alpha public\nPeptides, and now also proteases.\n" + (fx["tmp_path"] / "profiles" / "public" / f"{fx['alpha_id']}.md").write_text( + edited, encoding="utf-8" + ) + + second = _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + assert "Created 1 profile revisions." in second.output + + revisions = db(lambda s: _revisions_for(s, fx["alpha_uuid"])) + assert len(revisions) == 4, "the edited file must produce a second revision" + + by_type: dict[str, list] = {} + for revision in revisions: + by_type.setdefault(revision.profile_type, []).append(revision) + assert len(by_type["public"]) == 2 + # Both the old and the new body are on record — this is history, not a replace. + assert {r.content for r in by_type["public"]} == {"# Alpha public\nPeptides.\n", edited} + # Control: the two files nobody touched are still at one revision each. + assert len(by_type["private"]) == 1 + assert len(by_type["memory"]) == 1 + + +def test_backfill_with_no_profile_directories_is_a_clean_no_op(db, runner, monkeypatch, tmp_path): + """Absence control for the fixture above: with no files on disk the command still + succeeds and creates nothing, so 'created 3' upthread is attributable to the files. + """ + empty = tmp_path / "empty" + empty.mkdir() + monkeypatch.chdir(empty) + + agent_id = f"{AGENT_PREFIX}lonely" + + async def _seed(session): + agent = await factories.make_agent(session, agent_id=agent_id, bot_name="ClitestLonelyBot") + return agent.id + + agent_uuid = db(_seed) + + result = _ok(runner.invoke(cli_app, ["backfill-profile-revisions"])) + assert "Created 0 profile revisions." in result.output + assert db(lambda s: _revisions_for(s, agent_uuid)) == [] + + +# =========================================================================== +# Harness self-check +# =========================================================================== + + +def test_cli_writes_to_the_test_database_not_the_configured_one(db, runner, orcid_stub, pg_url): + """The tests above would all pass against the wrong database. Prove the CLI's own + `_get_db()` resolved to the test DSN by reading the row back through the test + engine, and prove the ambient config really was pointing somewhere else.""" + from src import config + + assert config.get_settings().database_url == pg_url + + orcid = _orcid(f"resolve-{uuid.uuid4().hex[:6]}") + orcid_stub.set(orcid, name="Resolution Probe") + _ok(runner.invoke(cli_app, ["seed-profile", "--orcid", orcid, "--no-pipeline"])) + + assert db(lambda s: _user_by_orcid(s, orcid)) is not None diff --git a/tests/integration/test_cohort_admin.py b/tests/integration/test_cohort_admin.py new file mode 100644 index 0000000..0e9b482 --- /dev/null +++ b/tests/integration/test_cohort_admin.py @@ -0,0 +1,1120 @@ +"""Live integration tests for the cohort admin surface. + +Real ASGI requests, real Postgres, real Jinja templates. Covers the granular +topology control (.notes/cohort-system-v2.md §12), the audit trail (§4.1/§13.1), +the delete guard, and the rule that the gate never becomes access control (§6.2). +""" + +import base64 +import json +import uuid + +import pytest +from itsdangerous import TimestampSigner +from sqlalchemy import select + +from src.config import get_settings +from src.models import AgentRegistry, Cohort, CohortAuditEvent, CohortMembership +from tests import factories + +pytestmark = pytest.mark.integration + + +def _auth(user_id) -> dict: + """Forge the signed session cookie SessionMiddleware would issue.""" + signer = TimestampSigner(get_settings().secret_key) + data = base64.b64encode(json.dumps({"user_id": str(user_id)}).encode()) + return {"Cookie": f"copi-session={signer.sign(data).decode()}"} + + +@pytest.fixture +async def admin(db_session): + return await factories.make_user(db_session, is_admin=True, email="admin@example.org") + + +@pytest.fixture +async def roster(db_session): + """Three active agents, mirroring the real bot-name convention.""" + out = {} + for aid, bot in (("su", "SuBot"), ("wiseman", "WisemanBot"), ("cravatt", "CravattBot")): + user = await factories.make_user(db_session, email=f"{aid}@example.org") + out[aid] = await factories.make_agent( + db_session, user=user, agent_id=aid, bot_name=bot, + pi_name=f"PI {aid}", status="active", + ) + await db_session.flush() + return out + + +async def _cohort(db_session, name, admin, members=()): + c = Cohort(name=name, created_by=admin.id) + db_session.add(c) + await db_session.flush() + for aid in members: + db_session.add(CohortMembership(cohort_id=c.id, agent_id=aid, added_by=admin.id)) + await db_session.flush() + return c + + +# --- agent role editing (task 11) ------------------------------------------- + + +async def test_admin_can_set_agent_role(client, db_session, admin, roster, tmp_path, monkeypatch): + # org1 ships no prompts/roles/ tree, so give the validator a real second + # role the same way tests/unit/test_roles.py does: a tmp roles dir. + from src.agent import roles as roles_mod + d = tmp_path / "roles" / "scout_hub" + d.mkdir(parents=True) + (d / "role.toml").write_text('label = "Scout Hub"\n', encoding="utf-8") + monkeypatch.setattr(roles_mod, "ROLES_DIR", tmp_path / "roles") + agent = roster["su"] + assert agent.role == "pi_lab" + r = await client.post( + f"/admin/agents/{agent.id}/role", + data={"role": "scout_hub"}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + row = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.id == agent.id) + )).scalar_one() + assert row.role == "scout_hub" + + +async def test_setting_an_unknown_role_is_rejected_without_a_500( + client, db_session, admin, roster +): + agent = roster["su"] + r = await client.post( + f"/admin/agents/{agent.id}/role", + data={"role": "not-a-real-role"}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert "error" in r.headers["location"] + row = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.id == agent.id) + )).scalar_one() + assert row.role == "pi_lab", "an unknown role must never be persisted" + + +async def test_setting_agent_role_requires_admin(client, db_session, admin, roster): + agent = roster["su"] + plain = await factories.make_user(db_session, is_admin=False, email="plain2@example.org") + await db_session.flush() + r = await client.post( + f"/admin/agents/{agent.id}/role", + data={"role": "scout_hub"}, + headers=_auth(plain.id), + ) + assert r.status_code == 403 + row = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.id == agent.id) + )).scalar_one() + assert row.role == "pi_lab" + + +async def test_setting_agent_role_requires_login(client, db_session, roster): + agent = roster["su"] + r = await client.post( + f"/admin/agents/{agent.id}/role", + data={"role": "scout_hub"}, + ) + assert r.status_code == 302 + assert "/login" in r.headers["location"] + row = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.id == agent.id) + )).scalar_one() + assert row.role == "pi_lab" + + +async def test_agent_detail_page_shows_the_current_role(client, admin, roster): + agent = roster["su"] + r = await client.get(f"/admin/agents/{agent.id}", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "pi_lab" in r.text + + +async def test_topology_page_shows_each_agents_role(client, db_session, admin, roster): + # The matrix table (and therefore any per-agent role cell) only renders once + # there is at least one cohort — see the "No cohorts yet" empty state. + await _cohort(db_session, "wave", admin) + r = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "pi_lab" in r.text + + +# --- access control --------------------------------------------------------- + + +async def test_cohort_pages_require_login(client): + for path in ("/admin/cohorts", "/admin/cohorts/topology"): + r = await client.get(path) + assert r.status_code == 302, path + assert "/login" in r.headers["location"] + + +async def test_cohort_pages_require_admin(client, db_session): + plain = await factories.make_user(db_session, is_admin=False, email="plain@example.org") + await db_session.flush() + r = await client.get("/admin/cohorts", headers=_auth(plain.id)) + assert r.status_code == 403 + + +# --- list + create --------------------------------------------------------- + + +async def test_list_renders_with_no_cohorts(client, admin, monkeypatch): + # Hermetic: the banner's "OFF" premise is cohort_isolation_enabled at its + # default (False). Pin it rather than inherit whatever the deployed .env on + # this host sets (COHORT_ISOLATION_ENABLED=true) — get_settings() is a + # process-wide lru_cache, so patch the cached instance's attribute directly. + monkeypatch.setattr(get_settings(), "cohort_isolation_enabled", False) + r = await client.get("/admin/cohorts", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "No cohorts yet" in r.text + # The banner must state what is actually in force. + assert "Cohort isolation is OFF" in r.text + assert "forward-only" in r.text + + +async def test_create_writes_an_audit_event(client, db_session, admin): + r = await client.post( + "/admin/cohorts/create", + data={"name": "pilot-wave-1", "description": "first wave"}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + c = (await db_session.execute( + select(Cohort).where(Cohort.name == "pilot-wave-1") + )).scalar_one() + ev = (await db_session.execute( + select(CohortAuditEvent).where(CohortAuditEvent.cohort_id == c.id) + )).scalars().all() + assert [e.action for e in ev] == ["created"] + assert ev[0].actor_email == "admin@example.org" + + +async def test_create_rejects_a_bad_name(client, db_session, admin): + r = await client.post( + "/admin/cohorts/create", data={"name": "Not A Slug!"}, headers=_auth(admin.id) + ) + assert r.status_code == 302 and "error=Invalid+name" in r.headers["location"] + assert (await db_session.execute(select(Cohort))).scalars().all() == [] + + +async def test_create_rejects_a_duplicate_name(client, db_session, admin): + await _cohort(db_session, "dupe", admin) + r = await client.post( + "/admin/cohorts/create", data={"name": "dupe"}, headers=_auth(admin.id) + ) + assert "error=A+cohort+with+that+name" in r.headers["location"] + + +# --- membership ------------------------------------------------------------ + + +async def test_add_and_remove_agent_are_both_audited(client, db_session, admin, roster): + c = await _cohort(db_session, "wave", admin) + r = await client.post( + f"/admin/cohorts/{c.id}/add-agent", data={"agent_id": "su"}, headers=_auth(admin.id) + ) + assert r.status_code == 302 + assert (await db_session.execute( + select(CohortMembership).where(CohortMembership.cohort_id == c.id) + )).scalars().all() + + r = await client.post( + f"/admin/cohorts/{c.id}/remove-agent", data={"agent_id": "su"}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert (await db_session.execute( + select(CohortMembership).where(CohortMembership.cohort_id == c.id) + )).scalars().all() == [] + + actions = [e.action for e in (await db_session.execute( + select(CohortAuditEvent) + .where(CohortAuditEvent.cohort_id == c.id) + .order_by(CohortAuditEvent.created_at) + )).scalars().all()] + assert actions == ["agent_added", "agent_removed"] + + +async def test_add_unknown_agent_is_refused(client, db_session, admin): + c = await _cohort(db_session, "wave", admin) + r = await client.post( + f"/admin/cohorts/{c.id}/add-agent", data={"agent_id": "nobody"}, + headers=_auth(admin.id), + ) + assert "error=Unknown+agent" in r.headers["location"] + assert (await db_session.execute(select(CohortMembership))).scalars().all() == [] + + +async def test_add_duplicate_member_is_refused(client, db_session, admin, roster): + c = await _cohort(db_session, "wave", admin, members=["su"]) + r = await client.post( + f"/admin/cohorts/{c.id}/add-agent", data={"agent_id": "su"}, headers=_auth(admin.id) + ) + assert "already+a+member" in r.headers["location"] + + +# --- delete guard ---------------------------------------------------------- + + +async def test_delete_is_refused_while_members_exist(client, db_session, admin, roster): + c = await _cohort(db_session, "populated", admin, members=["su", "wiseman"]) + r = await client.post(f"/admin/cohorts/{c.id}/delete", headers=_auth(admin.id)) + assert r.status_code == 302 + assert "Remove+all+2+members" in r.headers["location"] + assert (await db_session.execute( + select(Cohort).where(Cohort.id == c.id) + )).scalar_one_or_none() is not None, "populated cohort must survive" + + +async def test_delete_succeeds_when_empty_and_keeps_the_trail(client, db_session, admin): + c = await _cohort(db_session, "empty", admin) + cid = c.id + r = await client.post(f"/admin/cohorts/{cid}/delete", headers=_auth(admin.id)) + assert r.status_code == 302 and "notice=Deleted" in r.headers["location"] + assert (await db_session.execute( + select(Cohort).where(Cohort.id == cid) + )).scalar_one_or_none() is None + trail = (await db_session.execute( + select(CohortAuditEvent).where(CohortAuditEvent.cohort_id == cid) + )).scalars().all() + assert "deleted" in {e.action for e in trail}, ( + "the audit trail must outlive the cohort row" + ) + assert all(e.cohort_name == "empty" for e in trail) + + +async def test_detail_hides_the_delete_button_when_populated( + client, db_session, admin, roster +): + c = await _cohort(db_session, "populated", admin, members=["su"]) + r = await client.get(f"/admin/cohorts/{c.id}", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "Remove all 1 members first" in r.text + + +# --- detail page + audit log ---------------------------------------------- + + +async def test_detail_renders_members_and_audit_log(client, db_session, admin, roster): + c = await _cohort(db_session, "wave", admin) + await client.post( + f"/admin/cohorts/{c.id}/add-agent", data={"agent_id": "su"}, headers=_auth(admin.id) + ) + r = await client.get(f"/admin/cohorts/{c.id}", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "SuBot" in r.text + assert "Audit log" in r.text + assert "agent_added" in r.text + assert "admin@example.org" in r.text + + +async def test_detail_404s_for_an_unknown_cohort(client, admin): + r = await client.get(f"/admin/cohorts/{uuid.uuid4()}", headers=_auth(admin.id)) + assert r.status_code == 404 + + +# --- topology matrix: granular control ------------------------------------ + + +async def test_topology_route_is_not_shadowed_by_the_uuid_path(client, admin): + """"/cohorts/topology" must resolve to the matrix, not to a UUID lookup.""" + r = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "Topology matrix" in r.text + + +async def test_topology_renders_a_cell_per_pair(client, db_session, admin, roster): + a = await _cohort(db_session, "alpha", admin, members=["su"]) + b = await _cohort(db_session, "beta", admin) + r = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert r.status_code == 200 + for c in (a, b): + for aid in ("su", "wiseman", "cravatt"): + assert f'value="{c.id}:{aid}"' in r.text, f"missing cell {c.name}/{aid}" + # Exactly the pre-existing membership is pre-ticked. Matched on the input tag + # itself: a bare count of "checked" also picks up the column-toggle script. + import re as _re + ticked = set(_re.findall( + r'name="cell"\s+value="([^"]+)"[^>]*?\bchecked\b', r.text, _re.S + )) + assert ticked == {f"{a.id}:su"}, ticked + + +def _hidden_marker_values(html: str, name: str) -> set[str]: + """Every ``value`` of a hidden ```` with the given ``name``.""" + import re + + out = set() + for tag in re.finditer(r"]*>", html): + t = tag.group(0) + if f'name="{name}"' not in t: + continue + value = re.search(r'value="([^"]*)"', t) + if value: + out.add(value.group(1)) + return out + + +def _all_cell_values(html: str) -> set[str]: + """Every ``value`` of a ``name="cell"`` checkbox, checked or not. + + Unlike ``_ticked_cells`` (below), this does not filter on ``checked`` — it + is used to assert the RENDERED cell set, not the pre-ticked one. + """ + import re + + out = set() + for tag in re.finditer(r"]*>", html): + t = tag.group(0) + if 'name="cell"' not in t: + continue + value = re.search(r'value="([^"]*)"', t) + if value: + out.add(value.group(1)) + return out + + +async def test_rendered_cells_are_exactly_the_marker_cross_product( + client, db_session, admin, roster +): + """Structural safety property (audit finding F5): the ``present_agent`` / + ``present_cohort`` markers the save route trusts to reconstruct ``rendered`` + must equal the ACTUAL cross product of cells the table drew, or a save can + silently delete memberships for a cell that was never shown (see + ``test_topology_save_only_touches_rendered_cells`` and friends above). + + The old per-cell ``present`` input was emitted INSIDE the nested cell loop, + so it was structurally impossible for a cell to render without a matching + marker. The markers now sit OUTSIDE the table (``cohort_topology.html:46-47``), + so that guarantee is no longer enforced by the template's structure — only by + convention. If a future change wraps the inner ``{% for c in cohorts %}`` + loop in a per-agent conditional (e.g. skip cohorts for a suspended agent), + the markers would still claim the full cross product while the table drew + fewer cells, and this test is what would catch the mismatch (it would fail + the other direction too: a cell rendered with no corresponding marker pair). + + A non-active agent is included deliberately — production has 3 ``pending`` + agents, and the "Acts on" column already special-cases non-active status + (a real conditional in that neighborhood), so a regression is plausible + exactly there. + """ + a = await _cohort(db_session, "alpha", admin, members=["su"]) + b = await _cohort(db_session, "beta", admin) + pending_user = await factories.make_user(db_session, email="pendingpi@example.org") + await factories.make_agent( + db_session, user=pending_user, agent_id="pendingagent", bot_name="PendingAgentBot", + pi_name="Pending PI", status="pending", + ) + + r = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert r.status_code == 200 + + present_agents = _hidden_marker_values(r.text, "present_agent") + present_cohorts = _hidden_marker_values(r.text, "present_cohort") + assert present_agents == {"su", "wiseman", "cravatt", "pendingagent"}, present_agents + assert present_cohorts == {str(a.id), str(b.id)}, present_cohorts + + expected_cross_product = { + f"{cid}:{aid}" for cid in present_cohorts for aid in present_agents + } + rendered_cells = _all_cell_values(r.text) + assert rendered_cells == expected_cross_product, ( + f"markers claim {len(expected_cross_product)} cells but the table drew " + f"{len(rendered_cells)}; missing={expected_cross_product - rendered_cells}, " + f"extra={rendered_cells - expected_cross_product}" + ) + + +async def test_topology_save_applies_adds_and_removes_in_one_pass( + client, db_session, admin, roster +): + a = await _cohort(db_session, "alpha", admin, members=["su"]) + b = await _cohort(db_session, "beta", admin) + # Drop su from alpha, add wiseman to alpha, add cravatt to beta — one save. + ticked = [f"{a.id}:wiseman", f"{b.id}:cravatt"] + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id), str(b.id)], + "present_agent": ["su", "wiseman", "cravatt"], + "cell": ticked, + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert "1+added" not in r.headers["location"] # 2 added, 1 removed + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(a.id), "wiseman"), (str(b.id), "cravatt")} + + +async def test_topology_save_audits_every_change(client, db_session, admin, roster): + a = await _cohort(db_session, "alpha", admin, members=["su"]) + await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id)], + "present_agent": ["su", "wiseman"], + "cell": [f"{a.id}:wiseman"], + }, + headers=_auth(admin.id), + ) + events = (await db_session.execute( + select(CohortAuditEvent).where(CohortAuditEvent.cohort_id == a.id) + )).scalars().all() + assert {e.action for e in events} == {"agent_added", "agent_removed"} + assert {e.agent_id for e in events} == {"su", "wiseman"} + + +async def test_topology_save_only_touches_rendered_cells( + client, db_session, admin, roster +): + """The data-loss guard: a partial form must not delete what it never showed.""" + a = await _cohort(db_session, "alpha", admin, members=["su"]) + b = await _cohort(db_session, "beta", admin, members=["cravatt"]) + # Submit ONLY alpha's cells, all unticked. Beta's membership must survive. + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id)], + "present_agent": ["su", "wiseman", "cravatt"], + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(b.id), "cravatt")}, ( + "a form that did not render beta must not delete beta's memberships" + ) + + +async def test_topology_save_rejects_an_empty_submission(client, db_session, admin, roster): + await _cohort(db_session, "alpha", admin, members=["su"]) + r = await client.post("/admin/cohorts/topology", data={}, headers=_auth(admin.id)) + assert "error=Nothing+to+save" in r.headers["location"] + assert len((await db_session.execute(select(CohortMembership))).scalars().all()) == 1 + + +async def test_topology_save_rejects_a_tick_outside_the_rendered_set( + client, db_session, admin, roster +): + a = await _cohort(db_session, "alpha", admin) + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id)], + "present_agent": ["su"], + "cell": [f"{a.id}:wiseman"], + }, + headers=_auth(admin.id), + ) + assert "error=Malformed+submission" in r.headers["location"] + assert (await db_session.execute(select(CohortMembership))).scalars().all() == [] + + +async def test_topology_save_ignores_unknown_ids(client, db_session, admin, roster): + """A stale form naming a deleted cohort or a removed agent writes nothing.""" + ghost = uuid.uuid4() + a = await _cohort(db_session, "alpha", admin) + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(ghost), str(a.id)], + "present_agent": ["su", "nobody"], + "cell": [f"{ghost}:su", f"{a.id}:nobody"], + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert (await db_session.execute(select(CohortMembership))).scalars().all() == [] + + +# --- gate preview --------------------------------------------------------- + + +async def test_preview_matches_the_engine_semantics( + client, db_session, admin, roster, monkeypatch +): + """The admin preview must be computed by the same function the engine uses.""" + from src.services.cohorts import compute_gates + + # Hermetic: "isolation off (the default)" is this test's stated premise. Pin + # it — see test_list_renders_with_no_cohorts for why ambient .env cannot be + # trusted here. + monkeypatch.setattr(get_settings(), "cohort_isolation_enabled", False) + + a = await _cohort(db_session, "alpha", admin, members=["su", "wiseman"]) + rows = [(a.id, "su"), (a.id, "wiseman")] + gates, _ = compute_gates( + membership_rows=rows, agent_ids=["cravatt", "su", "wiseman"], + isolation_enabled=True, policy="open", cohort_count=1, + ) + # Under policy=open the uncohorted agent is included in su's gate, so the two can + # actually converse (both directions). See the unit-level regression test. + assert gates["su"] == {"su", "wiseman", "cravatt"} + assert gates["cravatt"] is None + + r = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert r.status_code == 200 + # With isolation off (the default) every agent is unrestricted. + assert "gate off for this agent" in r.text or "Cohort isolation is OFF" in r.text + + +async def test_inactive_agent_is_labelled_not_unrestricted( + client, db_session, admin, roster +): + roster["cravatt"].status = "suspended" + await db_session.flush() + await _cohort(db_session, "alpha", admin, members=["su"]) + r = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert "not active — the engine will not load this agent" in r.text + + +# --- the gate is not access control -------------------------------------- + + +async def test_pi_facing_thread_view_is_never_cohort_filtered( + client, db_session, admin, roster +): + """A cohort must never change what a human can read (v2 §6.2). + + Two agents in different cohorts exchange messages; the admin discussion view + must still show both. + """ + run = await factories.make_simulation_run(db_session) + await _cohort(db_session, "alpha", admin, members=["su"]) + await _cohort(db_session, "beta", admin, members=["cravatt"]) + await factories.make_agent_message( + db_session, run=run, agent_id="su", content="from su", + channel_name="general", message_ts="100.1", + ) + await factories.make_agent_message( + db_session, run=run, agent_id="cravatt", content="from cravatt", + channel_name="general", message_ts="100.2", + ) + await db_session.flush() + r = await client.get("/admin/discussions", headers=_auth(admin.id)) + assert r.status_code == 200 + + +# --- §11: what takes effect immediately and what needs a restart ------------ + + +def _ticked_cells(html: str) -> set[str]: + """The cells the matrix rendered as already-checked. + + Matches one whole ```` tag at a time. A regex that let the ``checked`` + lookahead run past the end of the tag picks up the column-toggle JavaScript's + ``b.checked`` for whichever agent happens to render last. + """ + import re + + out = set() + for tag in re.finditer(r"]*>", html): + t = tag.group(0) + if 'name="cell"' not in t: + continue + value = re.search(r'value="([^"]*)"', t) + if value and re.search(r"\bchecked\b", t): + out.add(value.group(1)) + return out + + +def test_ticked_cells_helper_distinguishes_checked_from_unchecked(): + """Control for the helper the live test depends on. An extractor that returned + every cell (or none) would make the assertion below meaningless.""" + html = ( + '' + '' + '' + "" + ) + assert _ticked_cells(html) == {"c1:su"} + + +async def test_membership_is_live_but_settings_are_cached(client, db_session, admin, roster): + """§11's asymmetry, both halves, so neither can pass alone. + + A topology edit takes effect on the engine's next roster sync (~30s, no restart). + The flag and the policy do not, because get_settings() is lru_cached — that is why + the admin banner says "restart required" for those and not for membership. If the + caching half ever stops being true, the banner is lying. + """ + import os + + from src.config import Settings, get_settings + + c = await _cohort(db_session, "alpha", admin) + before = get_settings().cohort_isolation_enabled + + os.environ["COHORT_ISOLATION_ENABLED"] = "true" if not before else "false" + try: + assert get_settings().cohort_isolation_enabled is before, ( + "get_settings() is no longer cached — §11 and the admin banner's " + "'restart required' wording are both wrong" + ) + # Control: a FRESH Settings() DOES see the env var. Without this leg the + # assertion above is also satisfied by an env var that never took effect. + assert Settings().cohort_isolation_enabled is not before, ( + "control leg failed: the env var had no effect even on a fresh Settings(), " + "so the caching assertion above proves nothing" + ) + finally: + os.environ.pop("COHORT_ISOLATION_ENABLED", None) + + # Positive: a membership change IS visible to the very next request, no restart. + r = await client.post( + f"/admin/cohorts/{c.id}/add-agent", data={"agent_id": "su"}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + page = await client.get("/admin/cohorts/topology", headers=_auth(admin.id)) + assert page.status_code == 200 + assert _ticked_cells(page.text) == {f"{c.id}:su"}, ( + "the membership edit is not reflected in the matrix" + ) + + +async def test_matrix_save_is_one_transaction(client, db_session, admin, roster): + """A mid-loop commit would expose an empty topology to a concurrent recompute. + + The engine reads cohort_memberships in a separate session. A save that committed + per row would let a recompute landing between commits see a partial — or, at the + instant every delete has landed and no insert has, an EMPTY — topology, which + under policy=isolated silences the whole roster. Structural rather than timing + based on purpose: a sleep-and-race test would be flaky and would not say why. + """ + import inspect + + from src.routers import admin as admin_mod + + src = inspect.getsource(admin_mod.admin_cohort_topology_save) + assert src.count("await db.commit()") == 1, ( + "the matrix save must commit exactly once; a per-row commit exposes an " + "empty-topology window to a concurrent gate recompute" + ) + assert "for cell in sorted(rendered)" in src + assert src.index("for cell in sorted(rendered)") < src.index("await db.commit()"), ( + "the commit must come after the whole diff loop" + ) + + +async def test_matrix_save_never_touches_an_unrendered_cohort( + client, db_session, admin, roster +): + """The classic checkbox-matrix data-loss bug, asserted from the outside. + + A form that rendered only alpha's cells must not delete beta's memberships, even + though beta's rows are absent from the submission and therefore look "unticked". + """ + a = await _cohort(db_session, "alpha", admin, members=["su"]) + b = await _cohort(db_session, "beta", admin, members=["cravatt"]) + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id)], + "present_agent": ["su", "wiseman", "cravatt"], + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(b.id), "cravatt")}, ( + f"a form that did not render beta must not delete beta's memberships. " + f"rows={rows}" + ) + # Control: alpha's rendered-and-unticked cell WAS removed, so the diff did run. + assert (str(a.id), "su") not in rows, ( + "control leg failed: the save did nothing at all, so the beta assertion " + "above proves nothing" + ) + + +async def test_matrix_save_ignores_a_cell_for_a_deleted_cohort( + client, db_session, admin, roster +): + """A stale form must not resurrect or crash on a cohort that no longer exists.""" + a = await _cohort(db_session, "alpha", admin, members=["su"]) + ghost = uuid.uuid4() + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id), str(ghost)], + "present_agent": ["su", "wiseman"], + "cell": [f"{a.id}:su", f"{ghost}:wiseman"], + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302, "a stale cell must not 500" + + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(a.id), "su")}, f"the ghost cell was written: {rows}" + + +async def test_matrix_save_ignores_a_cell_for_an_unknown_agent( + client, db_session, admin, roster +): + """Same for an agent id that is not in AgentRegistry — a membership naming a + nonexistent agent would be invisible in the UI and would survive forever.""" + a = await _cohort(db_session, "alpha", admin) + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_cohort": [str(a.id)], + "present_agent": ["su", "nobody"], + "cell": [f"{a.id}:su", f"{a.id}:nobody"], + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(a.id), "su")}, f"an unknown agent id was written: {rows}" + + +# --- attribution and the not-found / no-op edges -------------------------- +# +# The handlers above are all entered by the tests before this line, but five +# branches inside them were never taken: the creator-name lookup on the list +# page, the empty-members path on the detail page, and the three +# missing-row paths (delete, add-agent, remove-agent). Each one is a place +# where the route can either 500 or silently do the wrong thing, so each gets +# its own test asserting the observable outcome. + + +async def test_list_attributes_each_cohort_to_its_creator(client, db_session, admin): + """The "Created by" column resolves created_by to a user name. + + A distinct creator (not the logged-in admin) is used deliberately: the page's + nav bar already prints the current user's name, so asserting on the admin's + own name would pass even if creator_map were never populated. + """ + creator = await factories.make_user( + db_session, name="Zelda Creator", email="zelda@example.org" + ) + await _cohort(db_session, "attributed", creator) + orphan = Cohort(name="orphaned", created_by=None) + db_session.add(orphan) + await db_session.flush() + + r = await client.get("/admin/cohorts", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "attributed" in r.text and "orphaned" in r.text + assert "Zelda Creator" in r.text, "created_by was never resolved to a name" + # A cohort whose creator row is gone (ondelete=SET NULL) must render without + # borrowing the other row's name. + orphan_row = next(f for f in r.text.split("orphaned" in f) + assert "Zelda Creator" not in orphan_row + + +async def test_detail_of_an_empty_cohort_renders_the_no_members_state( + client, db_session, admin, roster +): + """With no memberships there are no adders to look up, and the members table + is replaced by the empty state rather than rendering a headless table.""" + c = await _cohort(db_session, "empty", admin) + r = await client.get(f"/admin/cohorts/{c.id}", headers=_auth(admin.id)) + assert r.status_code == 200 + assert "No members yet" in r.text + # The add-agent picker still offers the whole active roster. + for bot in ("SuBot", "WisemanBot", "CravattBot"): + assert bot in r.text, f"{bot} missing from the picker" + + +async def test_deleting_an_unknown_cohort_is_a_404(client, db_session, admin): + """A double-submitted delete (or a stale bookmark) must not raise, and must not + claim to have deleted anything. + + This used to be a bare redirect to the list carrying neither ``error=`` nor + ``notice=``. The successful path redirects with ``notice=Deleted+cohort+{name}``, + so the silent version was the only outcome in the whole surface that reported + nothing whatsoever — a second submit of an already-processed delete just landed + back on the list. It is now a 404, the same answer ``add-agent`` and + ``remove-agent`` give for a missing cohort and the same answer + ``GET /admin/cohorts/{id}`` gives for this very id. + """ + ghost = uuid.uuid4() + r = await client.post(f"/admin/cohorts/{ghost}/delete", headers=_auth(admin.id)) + assert r.status_code == 404 + assert (await db_session.execute( + select(CohortAuditEvent).where(CohortAuditEvent.cohort_id == ghost) + )).scalars().all() == [], "a delete that deleted nothing must not be audited" + + +async def test_a_real_delete_still_reports_success(client, db_session, admin): + """Positive control for the 404 above: the same route, given a cohort that does + exist, still redirects with a notice. Without this, a handler that 404'd + unconditionally would pass the test above.""" + c = await _cohort(db_session, "realdelete", admin) + r = await client.post(f"/admin/cohorts/{c.id}/delete", headers=_auth(admin.id)) + assert r.status_code == 302 + assert "notice=Deleted+cohort+realdelete" in r.headers["location"] + + +async def test_adding_an_agent_to_an_unknown_cohort_is_a_404( + client, db_session, admin, roster +): + """The membership must not be created against a cohort id that does not exist: + there is no FK from cohort_memberships.agent_id, and an orphan row would be + invisible in every cohort view.""" + r = await client.post( + f"/admin/cohorts/{uuid.uuid4()}/add-agent", + data={"agent_id": "su"}, + headers=_auth(admin.id), + ) + assert r.status_code == 404 + assert (await db_session.execute(select(CohortMembership))).scalars().all() == [] + + +async def test_removing_an_agent_that_is_not_a_member_is_a_silent_no_op( + client, db_session, admin, roster +): + """A stale Remove button must neither 500 nor forge an audit event.""" + c = await _cohort(db_session, "wave", admin, members=["su"]) + r = await client.post( + f"/admin/cohorts/{c.id}/remove-agent", + data={"agent_id": "wiseman"}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert r.headers["location"] == f"/admin/cohorts/{c.id}" + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(c.id), "su")}, ( + f"removing a non-member touched the real membership: {rows}" + ) + assert (await db_session.execute( + select(CohortAuditEvent).where(CohortAuditEvent.cohort_id == c.id) + )).scalars().all() == [], "a removal that removed nothing must not be audited" + + +async def test_removing_an_agent_from_an_unknown_cohort_is_a_404( + client, db_session, admin, roster +): + """Same handler, cohort row missing too. + + This used to redirect to ``/admin/cohorts/{ghost}`` — a detail page that then + 404s on its own, so the admin spent two round trips to reach the same error. + The handler now answers 404 directly. Note this is *not* the same case as + ``test_removing_an_agent_that_is_not_a_member_is_a_silent_no_op`` above, where + the cohort exists and the redirect target is a real page. + """ + ghost = uuid.uuid4() + r = await client.post( + f"/admin/cohorts/{ghost}/remove-agent", + data={"agent_id": "su"}, + headers=_auth(admin.id), + ) + assert r.status_code == 404 + assert (await db_session.execute(select(CohortAuditEvent))).scalars().all() == [] + + +async def test_topology_save_round_trips_with_marker_payload(client, db_session, admin): + """The new payload adds and removes exactly the ticked/unticked cells.""" + c1 = await _cohort(db_session, "alpha-marker", admin) + c2 = await _cohort(db_session, "beta-marker", admin) + await factories.make_agent(db_session, agent_id="ta1", bot_name="Ta1Bot") + await factories.make_agent(db_session, agent_id="ta2", bot_name="Ta2Bot") + # Pre-existing membership that the save must REMOVE (unticked but rendered). + db_session.add(CohortMembership(cohort_id=c1.id, agent_id="ta2", added_by=admin.id)) + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data={ + "present_agent": ["ta1", "ta2"], + "present_cohort": [str(c1.id), str(c2.id)], + "cell": [f"{c1.id}:ta1"], + }, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert "1+added,+1+removed" in r.headers["location"], r.headers["location"] + + rows = { + (str(cid), aid) + for cid, aid in (await db_session.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + } + assert rows == {(str(c1.id), "ta1")} + + +async def test_a_form_omitting_a_column_cannot_delete_that_columns_memberships( + client, db_session, admin +): + """The stale-form data-loss guard survives the cross-product reconstruction.""" + c1 = await _cohort(db_session, "shown", admin) + c2 = await _cohort(db_session, "hidden", admin) + await factories.make_agent(db_session, agent_id="tb1", bot_name="Tb1Bot") + db_session.add(CohortMembership(cohort_id=c2.id, agent_id="tb1", added_by=admin.id)) + await db_session.commit() + + # c2 is NOT in present_cohort, so its cell was never rendered. + r = await client.post( + "/admin/cohorts/topology", + data={"present_agent": ["tb1"], "present_cohort": [str(c1.id)]}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + + survivors = { + (str(cid), aid) + for cid, aid in (await db_session.execute( + select(CohortMembership.cohort_id, CohortMembership.agent_id) + )).all() + } + assert survivors == {(str(c2.id), "tb1")}, "a hidden column's membership was deleted" + + +async def test_a_form_omitting_a_row_cannot_delete_that_rows_memberships( + client, db_session, admin +): + c1 = await _cohort(db_session, "only", admin) + await factories.make_agent(db_session, agent_id="tc1", bot_name="Tc1Bot") + await factories.make_agent(db_session, agent_id="tc2", bot_name="Tc2Bot") + db_session.add(CohortMembership(cohort_id=c1.id, agent_id="tc2", added_by=admin.id)) + await db_session.commit() + + r = await client.post( + "/admin/cohorts/topology", + data={"present_agent": ["tc1"], "present_cohort": [str(c1.id)]}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + + survivors = { + aid for (aid,) in (await db_session.execute( + select(CohortMembership.agent_id) + )).all() + } + assert survivors == {"tc2"}, "a hidden row's membership was deleted" + + +async def test_full_matrix_payload_stays_under_the_field_limit(client, db_session, admin): + """60x56 used to post 3,528 fields against Starlette's max_fields=1000.""" + cohorts = [] + for i in range(56): + c = Cohort(name=f"c{i:03d}", created_by=admin.id) + db_session.add(c) + cohorts.append(c) + await db_session.flush() + for i in range(60): + await factories.make_agent( + db_session, agent_id=f"td{i:03d}", bot_name=f"Td{i:03d}Bot" + ) + await db_session.commit() + + present_agents = [f"td{i:03d}" for i in range(60)] + present_cohorts = [str(c.id) for c in cohorts] + cell = [f"{cohorts[0].id}:td000"] + total_fields = len(present_agents) + len(present_cohorts) + len(cell) + assert total_fields == 117, f"expected 116 markers + 1 cell, got {total_fields}" + + r = await client.post( + "/admin/cohorts/topology", + data={"present_agent": present_agents, "present_cohort": present_cohorts, "cell": cell}, + headers=_auth(admin.id), + ) + assert r.status_code == 302, r.text + assert "1+added" in r.headers["location"] + + +async def test_a_payload_of_unknown_marker_ids_does_not_blow_up_or_delete_anything( + client, db_session, admin, roster +): + """Many garbage present_cohort/present_agent ids must not explode the + cross product and must not be treated as an empty (``Nothing to save``) or + malformed submission — they are simply inert, like any other stale id. + + ``rendered`` used to be built as the cross product of the RAW, unfiltered + marker sets, so a payload naming only ids that no longer exist made the + product multiplicative in attacker-controlled input: N garbage cohort ids + times M garbage agent ids, regardless of how few real rows exist. This + posts several hundred of each (a full-scale reproduction of the reported + bound — tens of thousands squared — would itself be irresponsible to run + in a test process) to confirm the request still completes quickly and + behaves as a harmless no-op, and that it does not disturb a real, + unrelated membership that was never named by any marker. + """ + # A real membership, named by nothing in the payload below, that must survive. + a = await _cohort(db_session, "untouched", admin, members=["su"]) + + ghost_cohorts = [str(uuid.uuid4()) for _ in range(500)] + ghost_agents = [f"ghost-agent-{i}" for i in range(500)] + r = await client.post( + "/admin/cohorts/topology", + data={"present_cohort": ghost_cohorts, "present_agent": ghost_agents}, + headers=_auth(admin.id), + ) + assert r.status_code == 302 + assert "error" not in r.headers["location"], ( + f"an all-unknown payload must be a harmless no-op, not an error: " + f"{r.headers['location']}" + ) + + rows = { + (str(m.cohort_id), m.agent_id) + for m in (await db_session.execute(select(CohortMembership))).scalars().all() + } + assert rows == {(str(a.id), "su")}, "an all-unknown-id payload touched real data" + + +async def test_every_cohort_route_answers_a_missing_cohort_the_same_way( + client, db_session, admin, roster +): + """The three mutating cohort routes once disagreed three ways about a cohort id + that does not exist: add-agent raised 404, delete redirected silently to the + list, remove-agent redirected to a detail page that 404s. They now all match + the GET detail page, which is the convention the rest of this module uses for a + missing path-addressed row (see ``admin_user_delete``, ``admin_approve_agent``, + ``admin_approve_access`` and friends). ``?error=`` redirects stay reserved for + bad form input against a cohort that really exists. + """ + ghost = uuid.uuid4() + calls = [ + ("GET", f"/admin/cohorts/{ghost}", None), + ("POST", f"/admin/cohorts/{ghost}/delete", {}), + ("POST", f"/admin/cohorts/{ghost}/add-agent", {"agent_id": "su"}), + ("POST", f"/admin/cohorts/{ghost}/remove-agent", {"agent_id": "su"}), + ] + codes = {} + for method, path, data in calls: + if method == "GET": + r = await client.get(path, headers=_auth(admin.id)) + else: + r = await client.post(path, data=data, headers=_auth(admin.id)) + codes[path.rsplit("/", 1)[-1]] = r.status_code + assert set(codes.values()) == {404}, f"routes still disagree: {codes}" + # And nothing was written on any of the four attempts. + assert (await db_session.execute(select(CohortMembership))).scalars().all() == [] + assert (await db_session.execute(select(CohortAuditEvent))).scalars().all() == [] diff --git a/tests/integration/test_cohort_engine_live.py b/tests/integration/test_cohort_engine_live.py new file mode 100644 index 0000000..5d87aa8 --- /dev/null +++ b/tests/integration/test_cohort_engine_live.py @@ -0,0 +1,1523 @@ +"""Live end-to-end cohort tests: real SimulationEngine, real Postgres, Slack OFF. + +The unit suite exercises the gate with fake DB objects. This module runs the real +engine methods against a real database with real rows committed, because that is +where the DB-primary conversation interface and the cohort gate actually meet: + +- `_recompute_allowed_sender_ids` issuing real SQL against cohort_memberships +- `_poll_inbound_from_db` ingesting real agent_messages rows written by "another + process", then a gated read filtering them per agent +- `_rebuild_state_from_db` + `_rebuild_agent_state` reconstructing threads on a + *resumed* run and the first recompute grandfathering them (v2 §8) +- `_record_topology_snapshot` actually writing a row — it is wrapped in try/except, + so a broken write would otherwise only log a warning +- `_sync_roster_from_db` adding/removing agents mid-run under an active gate +- the full topology matrix (v2 §5.2) evaluated through the engine, not the helper + +Slack is off (NullTransport) throughout: that is the configuration where the DB is +the sole conversation store, so the gate's correctness rests entirely on read-side +filtering with no second path to incidentally catch a miss (v2 §9.1). +""" + +import uuid + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.transport import NullTransport +from src.models import ( + COHORT_ACTION_TOPOLOGY_SNAPSHOT, + AgentMessage, + AgentRegistry, + Cohort, + CohortAuditEvent, + CohortMembership, + SimulationRun, +) +from src.visibility import VISIBILITY_COLLAB_PRIVATE, VISIBILITY_PUBLIC + +pytestmark = pytest.mark.integration + +AGENT_IDS = ("su", "wiseman", "cravatt", "lotz") + + +@pytest.fixture +async def live(engine, monkeypatch): + """A committing session factory plus a cleanup of everything we write. + + Deliberately NOT the rolled-back `db_session` fixture: the engine opens its own + sessions and commits, and the whole point here is to exercise that path. + """ + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for aid in AGENT_IDS: + db.add(AgentRegistry( + agent_id=aid, bot_name=f"{aid.capitalize()}Bot", + pi_name=f"PI {aid}", status="active", + )) + await db.commit() + + yield factory, run_id + + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute(delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(AGENT_IDS))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +def _engine(factory, run_id, agent_ids=AGENT_IDS): + """A real SimulationEngine with Slack off.""" + agents = [ + Agent(agent_id=a, bot_name=f"{a.capitalize()}Bot", pi_name=f"PI {a}") + for a in agent_ids + ] + eng = SimulationEngine( + agents=agents, + slack_clients={a: NullTransport(a) for a in agent_ids}, + budget_cap=0, + session_factory=factory, + simulation_run_id=run_id, + slack_enabled=False, + ) + eng.message_log.set_bot_name_map({f"{a}bot": a for a in agent_ids}) + eng._bot_name_to_id = {f"{a}bot": a for a in agent_ids} + # start() registers this; without it nothing reaches agent_messages and a test + # asserting on the persisted row would silently pass on an empty result. + eng.message_log.set_persist_callback(eng._enqueue_persist) + return eng + + +def _cfg(monkeypatch, *, enabled=True, policy="isolated", valve=3, delay=0.0): + """Override only the cohort knobs on the REAL Settings object. + + A SimpleNamespace would work for the gate alone, but these tests drive real + phases, which read a dozen unrelated settings. Copying the real object keeps + every other value authentic and means a new setting cannot break the harness. + """ + from src.config import get_settings as _real + + patched = _real().model_copy(update={ + "cohort_isolation_enabled": enabled, + "cohort_default_policy": policy, + "max_consecutive_reactive_turns": valve, + "turn_delay_seconds": delay, + }) + monkeypatch.setattr("src.agent.simulation.get_settings", lambda: patched) + monkeypatch.setattr("src.config.get_settings", lambda: patched) + + +async def _topology(factory, mapping): + """mapping: {cohort_name: [agent_id, ...]} — committed for real.""" + async with factory() as db: + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + for name, members in mapping.items(): + c = Cohort(name=name) + db.add(c) + await db.flush() + for aid in members: + db.add(CohortMembership(cohort_id=c.id, agent_id=aid)) + await db.commit() + + +async def _write_message(factory, run_id, **kw): + """A row written as if by another process (web app, second engine, backfill).""" + defaults = dict( + simulation_run_id=run_id, channel_id="C1", channel_name="general", + message_length=10, phase="new_post", visibility=VISIBILITY_PUBLIC, + is_bot=True, thread_ts=None, + ) + defaults.update(kw) + async with factory() as db: + db.add(AgentMessage(**defaults)) + await db.commit() + + +# A roster the size of the intended pilot. Kept separate from AGENT_IDS so the +# four-agent tests — which is most of this module — stay fast. +AGENT_IDS_20 = ( + "su", "wiseman", "cravatt", "lotz", "racki", "schultz", "wolan", "paegel", + "joseph", "ward", "lairson", "bollong", "shen", "chatterjee", "kelly", + "hull", "baran", "sharpless", "nolan", "wu", +) + + +@pytest.fixture +async def live20(engine): + """20 active agents, same contract as `live`.""" + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for aid in AGENT_IDS_20: + db.add(AgentRegistry( + agent_id=aid, bot_name=f"{aid.capitalize()}Bot", + pi_name=f"PI {aid}", status="active", + )) + await db.commit() + + yield factory, run_id + + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute(delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(AGENT_IDS_20))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +# =========================================================================== +# The topology matrix — every shape, both policies, through the real engine +# =========================================================================== + + +TOPOLOGIES = { + "empty": {}, + "one_empty_cohort": {"alpha": []}, + "single_solo": {"alpha": ["su"]}, + "one_pair": {"alpha": ["su", "wiseman"]}, + "two_disjoint_pairs": {"alpha": ["su", "wiseman"], "beta": ["cravatt", "lotz"]}, + "overlapping": {"alpha": ["su", "wiseman"], "beta": ["su", "cravatt"]}, + "one_big_cohort": {"alpha": list(AGENT_IDS)}, + "hub_in_all": { + "alpha": ["su", "wiseman"], "beta": ["su", "cravatt"], "gamma": ["su", "lotz"], + }, + "partial": {"alpha": ["su", "wiseman"]}, # cravatt + lotz uncohorted + "offline_member_only": {"alpha": ["ghost"]}, # member not on the roster +} + +EXPECTED_ISOLATED = { + # (topology, policy) -> {agent_id: expected gate} (None = unrestricted) + ("empty", "isolated"): "REFUSED", + ("one_empty_cohort", "isolated"): "REFUSED", + ("offline_member_only", "isolated"): "REFUSED", + ("single_solo", "isolated"): { + "su": {"su"}, "wiseman": set(), "cravatt": set(), "lotz": set(), + }, + ("one_pair", "isolated"): { + "su": {"su", "wiseman"}, "wiseman": {"su", "wiseman"}, + "cravatt": set(), "lotz": set(), + }, + ("two_disjoint_pairs", "isolated"): { + "su": {"su", "wiseman"}, "wiseman": {"su", "wiseman"}, + "cravatt": {"cravatt", "lotz"}, "lotz": {"cravatt", "lotz"}, + }, + ("overlapping", "isolated"): { + "su": {"su", "wiseman", "cravatt"}, "wiseman": {"su", "wiseman"}, + "cravatt": {"su", "cravatt"}, "lotz": set(), + }, + ("one_big_cohort", "isolated"): {a: set(AGENT_IDS) for a in AGENT_IDS}, + ("hub_in_all", "isolated"): { + "su": {"su", "wiseman", "cravatt", "lotz"}, + "wiseman": {"su", "wiseman"}, "cravatt": {"su", "cravatt"}, + "lotz": {"su", "lotz"}, + }, + ("partial", "isolated"): { + "su": {"su", "wiseman"}, "wiseman": {"su", "wiseman"}, + "cravatt": set(), "lotz": set(), + }, +} + +EXPECTED_OPEN = { + # policy="open": uncohorted agents are unrestricted (None), never silenced. + ("empty", "open"): {a: None for a in AGENT_IDS}, + ("one_empty_cohort", "open"): {a: None for a in AGENT_IDS}, + ("offline_member_only", "open"): {a: None for a in AGENT_IDS}, + ("single_solo", "open"): { + "su": {"su", "wiseman", "cravatt", "lotz"}, + "wiseman": None, "cravatt": None, "lotz": None, + }, + ("one_pair", "open"): { + "su": {"su", "wiseman", "cravatt", "lotz"}, + "wiseman": {"su", "wiseman", "cravatt", "lotz"}, + "cravatt": None, "lotz": None, + }, + ("two_disjoint_pairs", "open"): { + "su": {"su", "wiseman"}, "wiseman": {"su", "wiseman"}, + "cravatt": {"cravatt", "lotz"}, "lotz": {"cravatt", "lotz"}, + }, + ("overlapping", "open"): { + "su": {"su", "wiseman", "cravatt", "lotz"}, + "wiseman": {"su", "wiseman", "lotz"}, + "cravatt": {"su", "cravatt", "lotz"}, "lotz": None, + }, + ("one_big_cohort", "open"): {a: set(AGENT_IDS) for a in AGENT_IDS}, + ("hub_in_all", "open"): { # every agent is cohorted, so nothing to add + "su": {"su", "wiseman", "cravatt", "lotz"}, + "wiseman": {"su", "wiseman"}, "cravatt": {"su", "cravatt"}, + "lotz": {"su", "lotz"}, + }, + ("partial", "open"): { + "su": {"su", "wiseman", "cravatt", "lotz"}, + "wiseman": {"su", "wiseman", "cravatt", "lotz"}, + "cravatt": None, "lotz": None, + }, +} + + +@pytest.mark.parametrize("name", sorted(TOPOLOGIES)) +@pytest.mark.parametrize("policy", ["open", "isolated"]) +async def test_topology_matrix_through_the_real_engine( + live, monkeypatch, name, policy +): + """Every topology shape x both policies, computed from real SQL.""" + factory, run_id = live + await _topology(factory, TOPOLOGIES[name]) + _cfg(monkeypatch, enabled=True, policy=policy) + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + expected = (EXPECTED_OPEN if policy == "open" else EXPECTED_ISOLATED)[(name, policy)] + if expected == "REFUSED": + assert eng._cohort_preflight_error is not None, ( + f"{name}/{policy} would silence the roster and must be refused" + ) + assert all(a.allowed_sender_ids is None for a in eng.agents.values()) + return + + assert eng._cohort_preflight_error is None, eng._cohort_preflight_error + actual = {aid: a.allowed_sender_ids for aid, a in eng.agents.items()} + assert actual == expected, f"{name}/{policy}: {actual} != {expected}" + + +@pytest.mark.parametrize("name", sorted(TOPOLOGIES)) +async def test_isolation_disabled_is_always_a_no_op(live, monkeypatch, name): + """Whatever the topology, the flag off means no filtering at all.""" + factory, run_id = live + await _topology(factory, TOPOLOGIES[name]) + _cfg(monkeypatch, enabled=False) + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert all(a.allowed_sender_ids is None for a in eng.agents.values()) + assert eng._cohort_gate_active is False + + +async def test_gate_relation_is_symmetric_for_every_topology(live, monkeypatch): + """If A may act on B then B may act on A — a shared cohort is symmetric, and + an asymmetric gate would let one side monologue.""" + factory, run_id = live + for name, mapping in TOPOLOGIES.items(): + if not any(mapping.values()): + continue + await _topology(factory, mapping) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + def _may_act(viewer, target_id): + """None means unrestricted, so it may act on anyone.""" + g = viewer.allowed_sender_ids + return True if g is None else target_id in g + + for a_id, a in eng.agents.items(): + for b_id, b in eng.agents.items(): + if a_id == b_id: + continue + # Deliberately NOT skipping the None-vs-set case. The earlier version + # of this test skipped it, and that is precisely how it missed the + # policy=open asymmetry: an uncohorted agent (gate None) could act on + # a cohorted one, but not the reverse, so the two could never converse. + assert _may_act(a, b_id) == _may_act(b, a_id), ( + f"{name}: asymmetric gate between {a_id} and {b_id} — " + f"{a_id} gate={a.allowed_sender_ids}, {b_id} gate={b.allowed_sender_ids}" + ) + + +# =========================================================================== +# The DB conversation interface under an active gate +# =========================================================================== + + +async def test_db_ingestion_is_complete_and_reads_are_per_agent(live, monkeypatch): + """The path that matters most: rows written by another process are ingested + whole into the shared log, and only the per-agent read is filtered.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await _write_message(factory, run_id, agent_id="wiseman", sender_name="WisemanBot", + content="from a cohort-mate", message_ts="1000.0001", + posted_at=1000.0001) + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="from outside", message_ts="1000.0002", + posted_at=1000.0002) + await _write_message(factory, run_id, agent_id=None, sender_name="Dr PI", + content="from a human", message_ts="1000.0003", + posted_at=1000.0003, is_bot=False) + + await eng._poll_inbound_from_db() + + # Shared log is complete — ingestion is never gated (v2 §6.2). + assert len(eng.message_log) == 3, "ingestion must not drop anything" + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + visible = eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + ) + assert {e.content for e in visible} == {"from a cohort-mate", "from a human"} + + # The excluded agent's own read sees its own side, plus the human. + cr = eng.agents["cravatt"] + visible_cr = eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="cravatt", + allowed_sender_ids=cr.allowed_sender_ids, + ) + assert {e.content for e in visible_cr} == {"from a human"} + + +async def test_null_agent_id_bot_row_from_the_db_does_not_leak(live, monkeypatch): + """agent_messages.agent_id is nullable. A bot row with a NULL agent_id must not + pass the gate as a human once ingested.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await _write_message(factory, run_id, agent_id=None, sender_name="bot", + content="unattributable bot row", message_ts="1000.0009", + posted_at=1000.0009, is_bot=True) + await eng._poll_inbound_from_db() + assert len(eng.message_log) == 1, "the row is still ingested" + + su = eng.agents["su"] + visible = eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + ) + assert visible == [], "an unattributable bot row must fail closed" + + +async def test_private_channel_message_from_a_non_mate_is_visible(live, monkeypatch): + """A PI-created pairing outranks the cohort, end to end through the DB.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await _write_message( + factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="my angle on the refinement", message_ts="1000.0011", + posted_at=1000.0011, channel_name="collab-priv-su-cravatt", + visibility=VISIBILITY_COLLAB_PRIVATE, + ) + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"collab-priv-su-cravatt"} + visible = eng.message_log.get_new_top_level_posts( + since=0, channels=su.state.subscribed_channels, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + ) + assert [e.content for e in visible] == ["my angle on the refinement"] + + +async def test_resumed_run_rebuild_then_first_recompute_grandfathers(live, monkeypatch): + """The §8 path that only exists on a restart, exercised for real. + + Messages from a previous process are in the DB. The rebuild reconstructs the + thread cohort-blind; the first recompute must mark it grandfathered. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + + # A thread from "before the restart": su's root + cravatt's reply. + await _write_message(factory, run_id, agent_id="su", sender_name="SuBot", + content="root post @CravattBot", message_ts="1000.0021", + posted_at=1000.0021) + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="replying", message_ts="1000.0022", + posted_at=1000.0022, thread_ts="1000.0021") + + eng = _engine(factory, run_id) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + + su = eng.agents["su"] + assert su.allowed_sender_ids is None, "the rebuild is gate-blind by construction" + threads = su.state.active_threads + assert threads, "the rebuild must reconstruct the thread" + t = next(iter(threads.values())) + assert t.other_agent_id == "cravatt" + assert t.grandfathered is False + + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is True, ( + "the first recompute after a rebuild must grandfather inherited " + "cross-cohort threads" + ) + assert eng._owes_reply(su) is False, "and it must not win reactive priority" + + +async def test_topology_snapshot_is_actually_written(live, monkeypatch): + """_record_topology_snapshot is wrapped in try/except, so a broken write would + only log a warning. Prove a row lands, with the applied gate in it.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + await eng._record_topology_snapshot() + + async with factory() as db: + rows = (await db.execute( + select(CohortAuditEvent).where( + CohortAuditEvent.action == COHORT_ACTION_TOPOLOGY_SNAPSHOT, + CohortAuditEvent.simulation_run_id == run_id, + ) + )).scalars().all() + assert len(rows) == 1, "no snapshot row was written" + topo = rows[0].topology + assert topo["cohort_default_policy"] == "isolated" + assert topo["agents"]["su"] == ["su", "wiseman"] + assert topo["agents"]["cravatt"] == [] + assert "counters" in topo + + +async def test_mid_run_topology_change_snapshots_again(live, monkeypatch): + """A topology edited during a run must leave a second snapshot, so the run's + output stays attributable to every configuration it ran under.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() # first signature + await eng._record_topology_snapshot() # startup snapshot + + await _topology(factory, {"alpha": ["su", "wiseman", "cravatt"]}) + await eng._recompute_allowed_sender_ids() # signature changes -> snapshots + + async with factory() as db: + rows = (await db.execute( + select(CohortAuditEvent) + .where(CohortAuditEvent.action == COHORT_ACTION_TOPOLOGY_SNAPSHOT) + .order_by(CohortAuditEvent.created_at) + )).scalars().all() + assert len(rows) >= 2, "a mid-run change must be recorded" + assert rows[-1].topology["agents"]["su"] == ["cravatt", "su", "wiseman"] + + +async def test_membership_change_takes_effect_without_restart(live, monkeypatch): + """The live-edit promise, against real SQL.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su"} + + await _topology(factory, {"alpha": ["su", "cravatt"]}) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su", "cravatt"} + + +async def test_roster_change_under_an_active_gate(live, monkeypatch): + """An agent deactivated mid-run must leave the roster and the gate cleanly.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman", "cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert "cravatt" in eng.agents + + async with factory() as db: + reg = (await db.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == "cravatt") + )).scalar_one() + reg.status = "suspended" + await db.commit() + + eng._last_roster_poll = 0.0 # force the throttle open + await eng._sync_roster_from_db() + assert "cravatt" not in eng.agents, "suspended agent must leave the roster" + # The remaining agents' gates still name cravatt (it is a cohort member), which + # is inert: it is not a live sender. Pinned so the behaviour is deliberate. + assert "cravatt" in eng.agents["su"].allowed_sender_ids + + +async def test_gate_survives_a_membership_row_for_an_unknown_agent(live, monkeypatch): + """A membership naming an agent that is not on the roster must not crash or + silence anyone.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "ghost-agent"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert eng._cohort_preflight_error is None + assert eng.agents["su"].allowed_sender_ids == {"su", "ghost-agent"} + assert eng.agents["wiseman"].allowed_sender_ids == set() + + +# =========================================================================== +# A real turn, with a faked LLM: does the gate actually reach the prompt? +# =========================================================================== + + +async def test_phase2_prompt_omits_non_cohort_posts(live, monkeypatch): + """The claim the whole feature rests on, verified at the LLM boundary. + + Phase 2 is the one batched Sonnet call per turn, and its prompt is where the + token saving is either real or imaginary. Drive a real Phase 2 with a scripted + LLM and assert the excluded agent's content never reaches the prompt, while the + cohort-mate's and the human's do. + """ + from tests.fakes import FakeAnthropic + + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + + fake = FakeAnthropic(['{"selected_post_ids": []}']) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await _write_message(factory, run_id, agent_id="wiseman", sender_name="WisemanBot", + content="MATE-CONTENT spatial multiomics", + message_ts="1000.0031", posted_at=1000.0031) + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="EXCLUDED-CONTENT chemoproteomics", + message_ts="1000.0032", posted_at=1000.0032) + await _write_message(factory, run_id, agent_id=None, sender_name="Dr PI", + content="HUMAN-CONTENT please collaborate", + message_ts="1000.0033", posted_at=1000.0033, is_bot=False) + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + su.state.last_seen_cursor = 0.0 + await eng._phase2_scan_filter(su) + + assert fake.calls, "Phase 2 should have made exactly one LLM call" + prompt = repr(fake.calls[0]) + assert "MATE-CONTENT" in prompt, "a cohort-mate's post must reach the prompt" + assert "HUMAN-CONTENT" in prompt, "a human's post must always reach the prompt" + assert "EXCLUDED-CONTENT" not in prompt, ( + "a non-cohort post reached the Phase 2 prompt — the gate is not saving " + "the tokens it claims to" + ) + + +async def test_phase2_makes_no_llm_call_when_everything_is_filtered(live, monkeypatch): + """When the only new posts are from excluded agents there is nothing to scan, + so the Sonnet call is skipped entirely — the actual saving.""" + from tests.fakes import FakeAnthropic + + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + fake = FakeAnthropic(['{"selected_post_ids": []}']) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="only excluded traffic", message_ts="1000.0041", + posted_at=1000.0041) + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + su.state.last_seen_cursor = 0.0 + await eng._phase2_scan_filter(su) + assert fake.calls == [], "no scannable posts must mean no LLM call" + + +async def test_phase3_does_not_activate_a_thread_from_a_non_cohort_tag(live, monkeypatch): + """Phase 3 is pure bookkeeping, but activating a thread with an excluded agent + would commit a thread slot and then drive Phase 4 spend.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="hey @SuBot want to work together?", + message_ts="1000.0051", posted_at=1000.0051) + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + su.state.last_seen_cursor = 0.0 + eng._phase3_activate_threads(su) + assert su.state.active_threads == {}, ( + "a tag from an excluded agent must not open a thread" + ) + + +async def test_phase3_does_activate_for_a_cohort_mate(live, monkeypatch): + """The same path must still work for a permitted sender — proving the previous + test is measuring the gate and not a broken Phase 3.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await _write_message(factory, run_id, agent_id="wiseman", sender_name="WisemanBot", + content="hey @SuBot want to work together?", + message_ts="1000.0061", posted_at=1000.0061) + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + su.state.last_seen_cursor = 0.0 + eng._phase3_activate_threads(su) + assert su.state.active_threads, "a cohort-mate's tag must still open a thread" + + +async def test_outbound_post_strips_a_cross_cohort_mention_for_real(live, monkeypatch): + """_post_message is the choke point, so drive it and read the persisted row.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + await eng._post_message( + "su", "general", "Good idea @WisemanBot — and @CravattBot too?" + ) + await eng._flush_persisted() + + async with factory() as db: + rows = (await db.execute( + select(AgentMessage).where( + AgentMessage.simulation_run_id == run_id, + AgentMessage.agent_id == "su", + ) + )).scalars().all() + assert len(rows) == 1 + content = rows[0].content + assert "@WisemanBot" in content, "a cohort-mate mention must survive" + assert "CravattBot" not in content, "a cross-cohort mention must be stripped" + assert eng._cohort_tags_stripped.get("su") == 1 + + +async def test_grandfathered_thread_still_gets_a_phase4_reply(live, monkeypatch): + """§8's central promise, driven end to end rather than asserted structurally. + + A thread whose partner has left the cohort must still be answered — abandoning + it mid-flight wastes every call already spent — while losing reactive priority. + """ + from tests.fakes import FakeAnthropic + + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + + fake = FakeAnthropic(["Happy to wrap this up."]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + # An open thread with an agent who is now outside the cohort. + await _write_message(factory, run_id, agent_id="su", sender_name="SuBot", + content="root post", message_ts="1000.0071", + posted_at=1000.0071) + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="a reply that deserves an answer", + message_ts="1000.0072", posted_at=1000.0072, + thread_ts="1000.0071") + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + su.state.last_seen_cursor = 0.0 + from src.agent.state import ThreadState + su.state.active_threads["1000.0071"] = ThreadState( + thread_id="1000.0071", channel="general", other_agent_id="cravatt", + message_count=2, + ) + await eng._recompute_allowed_sender_ids() + thread = su.state.active_threads["1000.0071"] + assert thread.grandfathered is True + + # It must not win reactive priority... + assert eng._owes_reply(su) is False + # ...but Phase 4 must still pick it up and reply. + replied = await eng._phase4_reply_threads(su) + assert "1000.0071" in replied, ( + "a grandfathered thread must still be answered so it can conclude" + ) + assert fake.calls, "Phase 4 should have called the LLM for the grandfathered thread" + + +async def test_pi_dm_path_is_unaffected_by_any_topology(live, monkeypatch): + """PI DMs bypass MessageLog entirely (_poll_pi_dms_from_db -> PIHandler), so no + cohort configuration may suppress them.""" + from src.models import PiDmMessage + + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + # su is maximally gated: only itself. + assert eng.agents["su"].allowed_sender_ids == {"su"} + + handled = [] + + class _Handler: + async def handle_dm(self, agent_id, pi_user_id, content): + handled.append((agent_id, content)) + + eng._pi_handler = _Handler() + + async with factory() as db: + db.add(PiDmMessage( + simulation_run_id=run_id, agent_id="su", pi_user_id="Uweb", + direction="inbound", content="please prioritise the immunology angle", + ts="1000.0081", + )) + await db.commit() + + await eng._poll_pi_dms_from_db() + assert handled == [("su", "please prioritise the immunology angle")], ( + "a PI DM must reach the agent under every cohort configuration" + ) + assert eng.agents["su"].state.has_pi_directive is True + + +# =========================================================================== +# Concurrency: membership writes must be atomic +# =========================================================================== + + +async def test_matrix_save_writes_memberships_atomically(live, monkeypatch): + """A wipe committed separately from the re-insert transiently opens the gate. + + Measured: with a two-transaction writer, ~57% of concurrent gate recomputes + landed in the preflight-refused state under `policy="isolated"` — i.e. the gate + was fully OPEN for those ticks — because a reader in the gap sees zero + memberships and the preflight correctly (but unhelpfully) refuses. With a + single-transaction writer it was 0 of ~4,500. + + The shipped `/admin/cohorts/topology` route is safe because it accumulates every + add and delete and commits once. This test pins that, because the dependency is + invisible: a future bulk importer that truncates and then inserts would silently + un-gate the roster about half the time. + """ + import inspect + + from src.routers import admin + + src = inspect.getsource(admin.admin_cohort_topology_save) + # Exactly one commit, and it is the last statement of the write path. + assert src.count("await db.commit()") == 1, ( + "the matrix save must commit exactly once; a mid-loop commit exposes an " + "empty-topology window to any concurrent gate recompute" + ) + body_after_loop = src[src.rindex("for cell in sorted(rendered):"):] + assert body_after_loop.index("await db.commit()") > body_after_loop.rindex( + "COHORT_ACTION_AGENT_REMOVED" + ), "the commit must come after every add/remove has been staged" + + +async def test_empty_topology_fails_open_not_closed(live, monkeypatch): + """If a reader ever does see an empty topology mid-write, the outcome must be + 'everyone unrestricted', never 'everyone silenced'.""" + factory, run_id = live + await _topology(factory, {"alpha": []}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert eng._cohort_preflight_error is not None + assert all(a.allowed_sender_ids is None for a in eng.agents.values()), ( + "an empty topology must fail OPEN — a transient write window must never " + "silence the roster" + ) + + +async def test_post_message_stamps_private_channel_visibility(live, monkeypatch): + """A message posted into a collab_private channel must persist as collab_private. + + Regression for a defect that made the §7 exemption dead code. `_post_message` + omitted `visibility` when constructing the LogEntry, so every agent-authored + message defaulted to "public" — even in a PI-created refinement channel. The gate's + private-channel exemption reads that field, so it never fired: two agents in + different cohorts could not converse in the channel the PI made for them. + + The existing §7 test could not catch this because it writes the AgentMessage row + directly with the visibility already set, exercising only the read path. This one + goes through `_post_message` and reads back what actually landed. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + priv = "collab-priv-su-cravatt" + eng._channel_visibility[priv] = VISIBILITY_COLLAB_PRIVATE + eng._channel_id_map[priv] = f"local:{priv}" + + await eng._post_message("cravatt", priv, "my angle on the refinement") + await eng._post_message("su", "general", "a public post") + await eng._flush_persisted() + + async with factory() as db: + rows = { + r.channel_name: r.visibility + for r in (await db.execute( + select(AgentMessage).where(AgentMessage.simulation_run_id == run_id) + )).scalars().all() + } + assert rows[priv] == VISIBILITY_COLLAB_PRIVATE, ( + "a message posted into a collab_private channel persisted as " + f"{rows[priv]!r} — the §7 exemption keys on this field and would never fire" + ) + assert rows["general"] == VISIBILITY_PUBLIC + + # And the exemption now actually fires: su is maximally gated, yet sees the message. + su = eng.agents["su"] + su.state.subscribed_channels = {priv} + visible = eng.message_log.get_new_top_level_posts( + since=0, channels={priv}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + ) + assert [e.content for e in visible] == ["my angle on the refinement"] + + +# =========================================================================== +# Scale + roster churn (v2 §14.6) +# =========================================================================== + + +async def test_gate_is_correct_and_affordable_at_20_agents(live20, monkeypatch): + """The pilot-scale recompute, with correctness as the control. + + A timing bound on its own is satisfied by a recompute that returns empty gates + instantly — which is the failure mode that actually matters here, since an empty + gate silences an agent. So the same gates are checked for being non-empty, + symmetric, and self-inclusive before the timing assertion runs. + """ + import statistics + import time + + factory, run_id = live20 + # 100 cohorts x 5 members, rotating through the roster: dense overlap, 500 rows. + mapping = { + f"c{i:03d}": [AGENT_IDS_20[(i * 5 + j) % 20] for j in range(5)] + for i in range(100) + } + await _topology(factory, mapping) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id, agent_ids=AGENT_IDS_20) + + timings = [] + for _ in range(10): + t0 = time.perf_counter() + await eng._recompute_allowed_sender_ids() + timings.append(time.perf_counter() - t0) + + gates = {a: x.allowed_sender_ids for a, x in eng.agents.items()} + assert len(gates) == 20 + assert eng._cohort_preflight_error is None + assert all(g for g in gates.values()), ( + f"an agent ended up with an empty gate: " + f"{[a for a, g in gates.items() if not g]}" + ) + for a, ga in gates.items(): + assert a in ga, f"{a} cannot see its own messages" + for b in ga: + assert a in gates[b], f"asymmetric gate {a} -> {b}" + + p50 = statistics.median(timings) + assert p50 < 0.5, ( + f"recompute p50 {p50 * 1000:.0f}ms at 100 cohorts / 500 memberships / 20 agents" + ) + + +async def test_deactivating_an_agent_mid_run_updates_roster_and_gate(live20, monkeypatch): + """A suspended agent leaves the live roster on the next sync, and the surviving + agents keep working gates rather than being reset to None or empty.""" + factory, run_id = live20 + await _topology( + factory, {"alpha": list(AGENT_IDS_20[:10]), "beta": list(AGENT_IDS_20[10:])} + ) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id, agent_ids=AGENT_IDS_20) + await eng._recompute_allowed_sender_ids() + assert "wu" in eng.agents + assert "wu" in eng.agents["nolan"].allowed_sender_ids + + async with factory() as db: + reg = (await db.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == "wu") + )).scalar_one() + reg.status = "suspended" + await db.commit() + + eng._last_roster_poll = 0.0 + await eng._sync_roster_from_db() + + assert "wu" not in eng.agents, "a suspended agent must leave the live roster" + # Control: the remaining agents still have real gates. A sync that wiped every gate + # would also satisfy the assertion above. + assert eng.agents["su"].allowed_sender_ids, "the gate was cleared by the sync" + assert "wiseman" in eng.agents["su"].allowed_sender_ids + assert "su" not in eng.agents["nolan"].allowed_sender_ids, ( + "cross-cohort isolation must survive a roster sync" + ) + + +async def test_activating_an_agent_mid_run_gives_it_a_gate(live20, monkeypatch): + """The mirror case. A newly activated agent must arrive WITH a gate applied, not + with allowed_sender_ids left at None — that would be a hole that opens itself.""" + factory, run_id = live20 + await _topology( + factory, {"alpha": list(AGENT_IDS_20[:10]), "beta": list(AGENT_IDS_20[10:])} + ) + _cfg(monkeypatch, enabled=True, policy="isolated") + # Start with 19 agents; "wu" exists in the registry but is not in the process. + eng = _engine(factory, run_id, agent_ids=AGENT_IDS_20[:19]) + await eng._recompute_allowed_sender_ids() + assert "wu" not in eng.agents + + eng._last_roster_poll = 0.0 + await eng._sync_roster_from_db() + + assert "wu" in eng.agents, "an active registry row must join the live roster" + gate = eng.agents["wu"].allowed_sender_ids + assert gate is not None, "a newly added agent arrived with NO gate — an open hole" + assert gate == set(AGENT_IDS_20[10:]), gate + + +# =========================================================================== +# §6.3 — forward-only cursor semantics +# =========================================================================== + + +async def test_filtering_is_forward_only(live, monkeypatch): + """A message the gate suppressed stays suppressed after the sender becomes a + cohort-mate, because the reading agent's cursor has already moved past it. + + This is documented in §6.3 as the reason a membership change never replays + backlog, and nothing tested it. The control is the third leg: a message posted + AFTER the change must be visible — otherwise a gate that simply never reopened + would satisfy the "no replay" assertion and the test would prove nothing. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="BEFORE the membership change", + message_ts="2000.0001", posted_at=2000.0001) + await eng._poll_inbound_from_db() + assert len(eng.message_log) == 1, "ingestion is never gated" + assert eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + ) == [], "precondition: the message must start out suppressed" + + # The turn advances the cursor past everything it was shown, as start() does. + su.state.last_seen_cursor = 2000.5 + + await _topology(factory, {"alpha": ["su", "cravatt"]}) + await eng._recompute_allowed_sender_ids() + assert "cravatt" in su.allowed_sender_ids, "the gate must have reopened" + + visible = eng.message_log.get_new_top_level_posts( + since=su.state.last_seen_cursor, channels={"general"}, + exclude_agent_id="su", allowed_sender_ids=su.allowed_sender_ids, + ) + assert [e.content for e in visible] == [], ( + "the backlog must NOT be replayed — filtering is forward-only" + ) + + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="AFTER the membership change", + message_ts="2001.0001", posted_at=2001.0001) + await eng._poll_inbound_from_db() + visible = eng.message_log.get_new_top_level_posts( + since=su.state.last_seen_cursor, channels={"general"}, + exclude_agent_id="su", allowed_sender_ids=su.allowed_sender_ids, + ) + assert [e.content for e in visible] == ["AFTER the membership change"], ( + "control leg failed: the gate never actually opened, so the forward-only " + "assertion above proves nothing" + ) + + +async def test_a_rewound_cursor_does_replay_and_the_gate_still_applies(live, monkeypatch): + """The other half of forward-only: the gate is not a one-shot stamp on the entry. + + If an agent's cursor is rewound (a resumed run rebuilding state, a bug, an admin), + the suppressed message becomes visible again — because the filter is evaluated per + read against the CURRENT gate, not baked into the log at ingest. The corollary + matters more: rewinding under a still-closed gate must NOT leak. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="suppressed at first", message_ts="2100.0001", + posted_at=2100.0001) + await eng._poll_inbound_from_db() + + def _read(since): + return [e.content for e in eng.message_log.get_new_top_level_posts( + since=since, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + )] + + # Rewound cursor, gate still closed: still nothing. + assert _read(0) == [], "rewinding the cursor must not bypass the gate" + + # Gate opens, cursor rewound: the entry is re-evaluated and now passes. This is + # what proves the filter is per-read rather than stamped at ingest. + await _topology(factory, {"alpha": ["su", "cravatt"]}) + await eng._recompute_allowed_sender_ids() + assert _read(0) == ["suppressed at first"] + + +# =========================================================================== +# §7 — the private-channel exemption, through the writers +# =========================================================================== + + +async def test_private_exemption_holds_for_every_write_path(live, monkeypatch): + """§7 driven through the write paths rather than by constructing the row. + + Rule B: the earlier §7 test wrote the AgentMessage row with `visibility` already + set, exercising only the read side — which is exactly how it missed that + `_post_message` never stamped the field at all. Every message here reaches the log + through a real writer. + + Three ways a private-channel message arrives: this engine posting it, another + process's row being ingested, and (below) a reply inside the channel. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su"}, "maximally gated" + + priv = "collab-priv-su-cravatt" + eng._channel_visibility[priv] = VISIBILITY_COLLAB_PRIVATE + eng._channel_id_map[priv] = f"local:{priv}" + + # (a) this engine posts into the private channel + await eng._post_message("cravatt", priv, "posted by the engine") + # (b) another process writes a private-channel row, then it is ingested + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="written by another process", message_ts="3000.0002", + posted_at=3000.0002, channel_name=priv, + channel_id=f"local:{priv}", + visibility=VISIBILITY_COLLAB_PRIVATE) + await eng._flush_persisted() + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {priv} + seen = {e.content for e in eng.message_log.get_new_top_level_posts( + since=0, channels={priv}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + )} + assert seen == {"posted by the engine", "written by another process"}, seen + + # (c) a threaded reply inside the private channel, checked through the GATED + # reply detector rather than get_thread_history (which is UNGATED by design, so + # asserting on it would prove nothing about the exemption). + root_ts = next( + e.ts for e in eng.message_log.get_new_top_level_posts( + since=0, channels={priv}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + ) if e.content == "posted by the engine" + ) + await eng._post_message("cravatt", priv, "a reply in-thread", thread_ts=root_ts) + await eng._flush_persisted() + assert eng.message_log.has_new_reply_from_other( + root_ts, "su", since=0.0, allowed_sender_ids=su.allowed_sender_ids, + ) is True, "a private-channel reply from a non-mate must still register (§7)" + + # Control: the SAME sender doing the SAME two things in a PUBLIC channel is + # filtered, so the exemption is scoped to the channel class and the gate is + # demonstrably still on. Without this leg the results above are also explained by + # the gate simply being off. + await eng._post_message("su", "general", "public root") + await eng._flush_persisted() + pub_root_ts = next( + e.ts for e in eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="cravatt", + allowed_sender_ids=None, + ) if e.content == "public root" + ) + await eng._post_message("cravatt", "general", "public post") + await eng._post_message("cravatt", "general", "public reply", thread_ts=pub_root_ts) + await eng._flush_persisted() + + pub = [e.content for e in eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids, + )] + assert pub == [], ( + f"control leg failed: the gate is not filtering public traffic ({pub}), so the " + "private-channel result above proves nothing" + ) + assert eng.message_log.has_new_reply_from_other( + pub_root_ts, "su", since=0.0, allowed_sender_ids=su.allowed_sender_ids, + ) is False, "control leg failed: a public non-mate reply must NOT register" + + +async def test_every_outbound_channel_class_is_stamped(live, monkeypatch): + """Regression guard for the defect that made §7 dead code. + + `_post_message` must stamp `visibility` for every channel class it can post into, + including one it has never seen — which must default to public rather than NULL. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + eng._channel_visibility["priv-a"] = VISIBILITY_COLLAB_PRIVATE + eng._channel_id_map["priv-a"] = "local:priv-a" + eng._channel_visibility["pub-a"] = VISIBILITY_PUBLIC + eng._channel_id_map["pub-a"] = "local:pub-a" + + await eng._post_message("su", "priv-a", "private") + await eng._post_message("su", "pub-a", "explicitly public") + await eng._post_message("su", "not-registered", "unregistered channel") + await eng._flush_persisted() + + async with factory() as db: + got = { + r.channel_name: r.visibility + for r in (await db.execute( + select(AgentMessage).where(AgentMessage.simulation_run_id == run_id) + )).scalars().all() + } + assert got["priv-a"] == VISIBILITY_COLLAB_PRIVATE + assert got["pub-a"] == VISIBILITY_PUBLIC + assert got["not-registered"] == VISIBILITY_PUBLIC, ( + "an unknown channel must default to public, not to NULL or a crash" + ) + # Control on the assertion set: the three values are not all the same, so a writer + # that hardcoded one constant could not pass. + assert len(set(got.values())) == 2, got + + +# =========================================================================== +# §8 grandfathering + §13.1 provenance, through the real startup path +# =========================================================================== + + +async def test_grandfathered_thread_concludes_but_loses_priority(live, monkeypatch): + """§8's two halves in one test, so neither can pass on its own. + + "Loses reactive priority" is an absence: _owes_reply going False would also be + satisfied by a thread that had simply gone quiet. So the same thread is asserted to + still receive a Phase 4 reply, with the LLM call as the witness — that is what + "concludes rather than being abandoned" means. + """ + from src.agent.state import ThreadState + from tests.fakes import FakeAnthropic + + factory, run_id = live + await _topology(factory, {"alpha": ["su", "cravatt"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + fake = FakeAnthropic(["Wrapping this up."]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + await _write_message(factory, run_id, agent_id="su", sender_name="SuBot", + content="root", message_ts="4000.0001", posted_at=4000.0001) + await _write_message(factory, run_id, agent_id="cravatt", sender_name="CravattBot", + content="a reply", message_ts="4000.0002", + posted_at=4000.0002, thread_ts="4000.0001") + await eng._poll_inbound_from_db() + + su = eng.agents["su"] + su.state.subscribed_channels = {"general"} + su.state.last_seen_cursor = 0.0 + su.state.active_threads["4000.0001"] = ThreadState( + thread_id="4000.0001", channel="general", other_agent_id="cravatt", + message_count=2, + ) + + # Positive leg BEFORE the split: the thread owes a reply and wins priority. + assert eng._owes_reply(su) is True, ( + "precondition failed: the thread does not owe a reply even in-cohort, so the " + "post-split assertion would prove nothing" + ) + + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + await eng._recompute_allowed_sender_ids() + thread = su.state.active_threads["4000.0001"] + assert thread.grandfathered is True, "the open cross-cohort thread must be marked" + + # Absence: it no longer jumps the queue. + assert eng._owes_reply(su) is False + + # Presence: Phase 4 still replies, so it can conclude. + replied = await eng._phase4_reply_threads(su) + assert "4000.0001" in replied, ( + "a grandfathered thread must still be answered so it can conclude" + ) + assert fake.calls, "Phase 4 made no LLM call — the thread stalled, not concluded" + + +async def test_start_computes_the_gate_and_records_a_snapshot(live, monkeypatch): + """§13.1 and the §8 pre-loop recompute through `start()` itself. + + Every other test in this module calls `_recompute_allowed_sender_ids()` directly, + so the ordering inside `start()` — gate computed and snapshot written BEFORE the + first turn — has never actually been exercised. `request_stop()` is triggered from + a setup step that runs after both, which is the least invasive way to let setup + complete and skip the loop. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + + original = eng._backfill_foa_cache + order = [] + + async def _stop_after_setup(): + # By the time this runs, start() has computed the gate and written the + # snapshot. Record what the gate looked like at that instant. + order.append({a: x.allowed_sender_ids for a, x in eng.agents.items()}) + eng.request_stop() + return await original() + + monkeypatch.setattr(eng, "_backfill_foa_cache", _stop_after_setup) + await eng.start() + + assert order, "setup never reached _backfill_foa_cache — start() aborted early" + assert order[0]["su"] == {"su", "wiseman"}, ( + f"the gate was not in force before the loop: {order[0]}" + ) + assert order[0]["cravatt"] == set() + + async with factory() as db: + snaps = (await db.execute( + select(CohortAuditEvent).where( + CohortAuditEvent.action == COHORT_ACTION_TOPOLOGY_SNAPSHOT, + CohortAuditEvent.simulation_run_id == run_id, + ) + )).scalars().all() + assert len(snaps) == 1, ( + f"start() must record exactly one startup snapshot, got {len(snaps)}" + ) + topo = snaps[0].topology + assert topo["agents"]["su"] == ["su", "wiseman"] + assert topo["cohort_default_policy"] == "isolated" + assert topo["cohort_isolation_enabled"] is True + + +async def test_start_records_a_snapshot_even_when_the_gate_is_off(live, monkeypatch): + """Control for the test above: provenance is unconditional. + + If the snapshot were only written when isolation is on, a run's output would be + unattributable in exactly the case an auditor cares about — "was the gate on?". + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=False) + eng = _engine(factory, run_id) + + original = eng._backfill_foa_cache + + async def _stop_after_setup(): + eng.request_stop() + return await original() + + monkeypatch.setattr(eng, "_backfill_foa_cache", _stop_after_setup) + await eng.start() + + async with factory() as db: + snaps = (await db.execute( + select(CohortAuditEvent).where( + CohortAuditEvent.action == COHORT_ACTION_TOPOLOGY_SNAPSHOT, + CohortAuditEvent.simulation_run_id == run_id, + ) + )).scalars().all() + assert len(snaps) == 1, "a gate-off run must still record its topology" + assert snaps[0].topology["cohort_isolation_enabled"] is False + + +# =========================================================================== +# §9 outbound tag hygiene, §10.3 the fairness valve +# =========================================================================== + + +# The strip's contract as data. su's gate is {su, wiseman}; cravatt and lotz are +# outside it. Each row is (input, expected output). +STRIP_CASES = [ + ("Great point @CravattBot, shall we?", "Great point, shall we?"), + ("@CravattBot hi", "hi"), + ("cc @CravattBot", "cc"), + ("(@CravattBot)", "()"), + ("a @CravattBot b @CravattBot c", "a b c"), + ("keep @WisemanBot, drop @CravattBot", "keep @WisemanBot, drop"), + ("self @SuBot stays", "self @SuBot stays"), + ("unknown @GhostBot stays", "unknown @GhostBot stays"), + ("mail a@cravattbot.example", "mail a@cravattbot.example"), + ("see http://x/@cravattbot", "see http://x/@cravattbot"), + ("@CravattBotly stays", "@CravattBotly stays"), + ("line1 @CravattBot\nline2", "line1\nline2"), + ("here:\n def f():\n return @CravattBot", + "here:\n def f():\n return"), + ("two outsiders @CravattBot @LotzBot", "two outsiders"), +] + + +@pytest.mark.parametrize("text,expected", STRIP_CASES, + ids=[c[0][:26] for c in STRIP_CASES]) +async def test_strip_cases(live, monkeypatch, text, expected): + """Every §9 behaviour as a row, so a failure names the surrounding, not the regex.""" + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su", "wiseman"} + assert eng._strip_disallowed_tags(text, eng.agents["su"]) == expected + + +def test_strip_cases_include_survival_rows(): + """Control for the table. + + At least one row must PRESERVE a cross-agent mention and at least one must remove + one. A table of removal-only rows would pass against a function that deleted every + @-mention; a table of preserve-only rows against one that did nothing. + """ + preserved = [t for t, exp in STRIP_CASES if "@" in exp] + removed = [t for t, exp in STRIP_CASES if "@CravattBot" in t and "@" not in exp] + assert preserved, "no row preserves a mention" + assert removed, "no row removes a mention" + assert any("@WisemanBot" in exp for _, exp in STRIP_CASES), ( + "no row proves a COHORT-MATE mention survives — the strip could be deleting " + "every mention and this table would not notice" + ) + + +async def test_strip_indentation_is_preserved(live, monkeypatch): + """Regression: the whitespace tidy-up must not reflow line-leading indentation. + + An earlier version normalised `(?m)^[ \\t]+`, which flattened the code blocks and + bullet lists agents put in messages. This runs on EVERY outbound message, so a + reflow would corrupt real content on a path that has nothing to do with cohorts. + """ + factory, run_id = live + await _topology(factory, {"alpha": ["su", "wiseman"]}) + _cfg(monkeypatch, enabled=True, policy="isolated") + eng = _engine(factory, run_id) + await eng._recompute_allowed_sender_ids() + + text = ( + "Proposal @CravattBot:\n" + "```python\n" + "def screen(hits):\n" + " for h in hits:\n" + " yield h\n" + "```\n" + "- first bullet\n" + " - nested bullet\n" + ) + out = eng._strip_disallowed_tags(text, eng.agents["su"]) + assert "@CravattBot" not in out, "the mention must still be stripped" + assert " for h in hits:" in out, "code-block indentation was reflowed" + assert " yield h" in out, "nested code indentation was reflowed" + assert " - nested bullet" in out, "list indentation was reflowed" + + +async def test_valve_holds_over_sustained_load(live20, monkeypatch): + """§10.3 at pilot scale over 200 selections. + + A fake clock is essential: with wall time every pick lands in the same instant, the + staleness weight clamps to 1.0 for everyone, and the proactive tier becomes uniform + — which would make the pair's share a property of the fake, not of the valve. + + Both bounds matter. The upper one is the valve doing its job; the LOWER one is the + control — a scheduler with no reactive tier at all would give the pair roughly + 2/20 of the turns and would pass an upper bound alone. + """ + import random + import types + + import src.agent.simulation as sim + from src.agent.state import ThreadState + + factory, run_id = live20 + await _topology(factory, {"alpha": list(AGENT_IDS_20)}) + _cfg(monkeypatch, enabled=True, policy="isolated", valve=3) + eng = _engine(factory, run_id, agent_ids=AGENT_IDS_20) + await eng._recompute_allowed_sender_ids() + + random.seed(20260730) + clock = [1000.0] + monkeypatch.setattr(sim, "time", types.SimpleNamespace(time=lambda: clock[0])) + + # Two agents locked in a perpetual exchange — the starvation scenario §10.3 exists + # for. The other 18 have nothing owed and can only be reached proactively. + for a, b in (("su", "wiseman"), ("wiseman", "su")): + eng.agents[a].state.active_threads[f"t-{a}"] = ThreadState( + thread_id=f"t-{a}", channel="general", other_agent_id=b, + has_pending_reply=True, + ) + + picks = [] + for _ in range(200): + got = eng._select_agent() + assert got is not None + picks.append(got.agent_id) + eng._last_llm_caller = got.agent_id + got.state.last_selected = clock[0] + clock[0] += 10.0 + + pair = sum(1 for p in picks if p in {"su", "wiseman"}) + assert pair <= 160, ( + f"the pair took {pair}/200 turns — the valve is not holding" + ) + assert pair >= 100, ( + f"the pair took only {pair}/200 — the reactive tier is not firing at all, so " + "the upper bound above proves nothing" + ) + # The valve's purpose: everyone else still gets to form new conversations. + others = set(picks) - {"su", "wiseman"} + assert len(others) >= 15, ( + f"only {len(others)} of the other 18 agents were ever selected: {sorted(others)}" + ) + assert eng._reactive_selections + eng._proactive_selections == 200 + assert eng._proactive_selections >= 40, ( + f"only {eng._proactive_selections} proactive picks in 200 — the valve should " + "force roughly one in four" + ) diff --git a/tests/integration/test_cohort_real_llm.py b/tests/integration/test_cohort_real_llm.py new file mode 100644 index 0000000..5ae03af --- /dev/null +++ b/tests/integration/test_cohort_real_llm.py @@ -0,0 +1,375 @@ +"""Cohort gate against the REAL Anthropic API. Skipped unless a key is present. + +Everything else in the cohort suite scripts the LLM. This module spends real tokens, +because two claims cannot be checked with a fake: + +1. A real model, given a Phase 2 prompt built under an active gate, cannot select or + reason about a post the gate removed — the post is not in the prompt at all. + A fake proves the prompt lacks the text; only a real call proves the model's + *output* is unaffected by the excluded content. +2. A real model asked to start a conversation will name a partner, and the outbound + strip must remove a cross-cohort mention from genuine model prose rather than from + a hand-written string. + +Cost control (the whole module is a handful of calls): +- ``max_tokens`` is capped hard. +- Prompts are the real ones, but the roster and history are minimal. +- Sonnet, not Opus, for the scan path — that is what Phase 2 uses anyway. +- One call per test, four tests. Roughly a cent at current prices. + +Run it with: + + docker compose exec -e ANTHROPIC_API_KEY=sk-ant-... \\ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_test \\ + app python -m pytest tests/integration/test_cohort_real_llm.py -v -m real_llm + +Without a key every test skips, so the default suite stays free and offline. +""" + +import os +import uuid + +import pytest + +from src.agent.agent import Agent +from src.agent.message_log import LogEntry, MessageLog +from src.visibility import VISIBILITY_PUBLIC + +pytestmark = [ + pytest.mark.integration, + pytest.mark.real_llm, + pytest.mark.skipif( + not os.environ.get("ANTHROPIC_API_KEY"), + reason="no ANTHROPIC_API_KEY — real-API tests are opt-in and cost money", + ), +] + +MAX_TOKENS = 300 + + +def _agent(agent_id="su", bot="SuBot"): + return Agent(agent_id=agent_id, bot_name=bot, pi_name=f"PI {agent_id}") + + +def _post(ts, agent_id, name, content): + return LogEntry( + ts=ts, channel="general", sender_agent_id=agent_id, sender_name=name, + content=content, thread_ts=None, posted_at=float(ts), is_bot=True, + visibility=VISIBILITY_PUBLIC, + ) + + +# A profile that makes the EXCLUDED post directly relevant and the INCLUDED post +# clearly irrelevant. Without this the scan has nothing to latch onto: an agent with +# no profile selects no posts either way, and the test passes vacuously — measured. +SU_PROFILE = """# Su Lab + +We run genome-scale CRISPR functional-genomics screens and build chemical-probe +pipelines. We are actively seeking collaborators in **activity-based protein +profiling** and **covalent ligand discovery** to turn screen hits into chemical +probes. We are NOT currently working on spatial transcriptomics or imaging. +""" + +# Irrelevant to SU_PROFILE — the post the gate lets through. +POST_IRRELEVANT = ( + "We built a spatial transcriptomics imaging atlas of tumour microenvironments " + "and are looking for an imaging-analysis partner." +) +# Directly relevant to SU_PROFILE — the post the gate removes. +POST_RELEVANT = ( + "We run activity-based protein profiling and want a functional-genomics " + "collaborator to pair covalent ligand discovery with CRISPR screen hits." +) + + +@pytest.fixture +def log(): + ml = MessageLog() + ml.set_bot_name_map({"subot": "su", "wisemanbot": "wiseman", "cravattbot": "cravatt"}) + ml.append(_post("1000.0001", "wiseman", "WisemanBot", POST_IRRELEVANT)) + ml.append(_post("1000.0002", "cravatt", "CravattBot", POST_RELEVANT)) + return ml + + +def _profiled_agent(): + a = _agent() + a._public_profile = SU_PROFILE # the cached-profile seam; avoids disk I/O + return a + + +async def _call(system_prompt, messages, model=None): + """One real API call. Sonnet (what Phase 2 uses) with a hard token cap.""" + from src.config import get_settings + from src.services import llm + + settings = get_settings() + return await llm.generate_agent_response( + system_prompt=system_prompt, + messages=messages, + model=model or settings.llm_agent_model_sonnet, + max_tokens=MAX_TOKENS, + log_meta={"agent_id": "su", "phase": "real_llm_audit"}, + ) + + +def _post_dicts(posts): + """Exactly the shape _phase2_scan_filter builds (note: content_snippet).""" + return [ + {"post_id": p.ts, "sender": p.sender_name, "channel": p.channel, + "content_snippet": p.content} + for p in posts + ] + + +def _selected_ids(response: str) -> set[str] | None: + """Parse selected_post_ids out of a real Phase 2 response.""" + import json + import re + + m = re.search(r"\{.*\}", response, re.S) + if not m: + return None + try: + data = json.loads(m.group(0)) + except json.JSONDecodeError: + return None + return set(map(str, data.get("selected_post_ids") or [])) + + +async def test_real_model_would_have_acted_on_the_post_the_gate_removes(log): + """The claim the whole feature rests on, measured on a real model. + + Two real Phase 2 calls with the same profile and the same log, differing only in + whether the gate is applied: + + - ungated, the model **selects** the excluded agent's post and explains why — + i.e. it would have opened a thread and spent Opus calls on it; + - gated, that post is absent from the prompt, so the model cannot select it. + + A fake LLM can only show the prompt lacks the text. Only a real call shows the + model's *decision* changes — which is what "the gate saves calls" actually means. + Asserting on both halves is deliberate: without the ungated leg, a model that + selects nothing regardless would make the gated leg pass for the wrong reason. + """ + a = _profiled_agent() + + ungated = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", allowed_sender_ids=None, + ) + assert {p.ts for p in ungated} == {"1000.0001", "1000.0002"} + sys_u, msg_u = a.build_phase2_scan_prompt(_post_dicts(ungated)) + assert POST_RELEVANT[:40] in sys_u + str(msg_u) + selected_ungated = _selected_ids(await _call(sys_u, msg_u)) + assert selected_ungated is not None, "real Phase 2 response did not parse" + assert "1000.0002" in selected_ungated, ( + "control leg failed: the model did not act on the relevant post even with the " + f"gate off, so the gated leg proves nothing. selected={selected_ungated}" + ) + + gated = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids={"su", "wiseman"}, + ) + assert {p.ts for p in gated} == {"1000.0001"} + sys_g, msg_g = a.build_phase2_scan_prompt(_post_dicts(gated)) + assert POST_RELEVANT[:40] not in sys_g + str(msg_g) + selected_gated = _selected_ids(await _call(sys_g, msg_g)) + assert selected_gated is not None, "real Phase 2 response did not parse" + assert "1000.0002" not in selected_gated, ( + "the model selected a post the gate removed — impossible unless the prompt " + f"leaked it. selected={selected_gated}" + ) + + assert selected_ungated != selected_gated, ( + "the gate produced no measurable change in the model's decision: " + f"{selected_ungated} vs {selected_gated}" + ) + + +async def test_real_model_prose_gets_its_cross_cohort_mention_stripped(monkeypatch): + """Ask a real model to write a post that tags a specific bot, then run the real + outbound strip over its actual prose.""" + import types + + import src.agent.simulation as sim + from src.agent.simulation import SimulationEngine + from src.agent.transport import NullTransport + + settings_ns = types.SimpleNamespace( + cohort_isolation_enabled=True, cohort_default_policy="isolated", + max_consecutive_reactive_turns=3, turn_delay_seconds=0.0, + ) + monkeypatch.setattr(sim, "get_settings", lambda: settings_ns) + + ids = ("su", "wiseman", "cravatt") + eng = SimulationEngine( + agents=[_agent(a, f"{a.capitalize()}Bot") for a in ids], + slack_clients={a: NullTransport(a) for a in ids}, + budget_cap=0, session_factory=None, slack_enabled=False, + ) + eng._bot_name_to_id = {f"{a}bot": a for a in ids} + su = eng.agents["su"] + su.allowed_sender_ids = {"su", "wiseman"} # cravatt is outside the cohort + + response = await _call( + "You are SuBot, a lab's research agent in a Slack channel. Reply with one " + "short paragraph only, no preamble.", + [{"role": "user", "content": + "Write a two-sentence Slack message proposing a collaboration. You must " + "mention both @WisemanBot and @CravattBot by name with the @ prefix."}], + ) + assert response, "the real API returned nothing" + + cleaned = eng._strip_disallowed_tags(response, su) + assert cleaned is not None + if "@CravattBot" in response: + assert "CravattBot" not in cleaned, ( + f"a cross-cohort mention survived the strip.\nmodel wrote: {response!r}\n" + f"after strip: {cleaned!r}" + ) + assert eng._cohort_tags_stripped.get("su", 0) >= 1 + else: + pytest.skip( + "the model did not produce an @CravattBot mention, so there was nothing " + f"to strip. Model output: {response!r}" + ) + if "@WisemanBot" in response: + assert "@WisemanBot" in cleaned, "a cohort-mate mention must survive" + + +async def test_real_scan_response_parses_under_an_active_gate(log): + """End-to-end shape check: a real Phase 2 response must still parse into post + ids the engine can act on, and can only name posts that survived the gate.""" + import json + import re + + a = _profiled_agent() + gated = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids={"su", "wiseman"}, + ) + allowed_ids = {p.ts for p in gated} + post_dicts = [ + {"post_id": p.ts, "sender": p.sender_name, "channel": p.channel, + "content_snippet": p.content} + for p in gated + ] + system, messages = a.build_phase2_scan_prompt(post_dicts) + response = await _call(system, messages) + assert response + + m = re.search(r"\{.*\}", response, re.S) + if not m: + pytest.skip(f"real model returned no JSON object: {response!r}") + data = json.loads(m.group(0)) + selected = data.get("selected_post_ids") or data.get("selected") or [] + assert set(map(str, selected)) <= allowed_ids | {""}, ( + f"the model selected a post id that was gated out: {selected} " + f"(allowed: {sorted(allowed_ids)})" + ) + + +async def test_real_model_acts_on_an_uncohorted_peer_under_open_policy(log): + """§5.2 with a real model: under `open`, a cohorted agent must be able to act on an + uncohorted one. + + This is the defect a real multi-turn run surfaced. The gate was asymmetric — the + uncohorted agent could react to anyone but appeared in nobody's mate set, so it + opened threads that were never answered. Every gate-computation test passed. + + Control: the same call with the uncohorted agent excluded from the gate must NOT + select the post, so a model that selects everything cannot pass. + """ + from src.services.cohorts import compute_gates + + c1 = uuid.uuid4() + gates, reason = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy="open", cohort_count=1, + ) + assert reason is None + assert gates["cravatt"] is None, "the uncohorted agent stays unrestricted" + assert "cravatt" in gates["su"], ( + f"precondition: the open-policy fix must be in place. su gate={gates['su']}" + ) + + a = _profiled_agent() + visible = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=gates["su"], + ) + assert {p.ts for p in visible} == {"1000.0001", "1000.0002"}, ( + "the uncohorted peer's post must reach su's prompt at all" + ) + selected = _selected_ids(await _call(*a.build_phase2_scan_prompt(_post_dicts(visible)))) + assert selected is not None, "real Phase 2 response did not parse" + assert "1000.0002" in selected, ( + f"the model did not act on the uncohorted peer's relevant post: {selected}" + ) + + # Control: exclude cravatt and the same model must not select it — it is not in + # the prompt to select. + gated = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids={"su", "wiseman"}, + ) + sel2 = _selected_ids(await _call(*a.build_phase2_scan_prompt(_post_dicts(gated)))) + assert sel2 is not None + assert "1000.0002" not in sel2, ( + f"control leg failed: the post was selected even when gated out ({sel2}), so " + "the prompt leaked it" + ) + + +async def test_real_model_cannot_reach_across_a_hub(log): + """Non-transitivity with a real model: A-B and B-C must not yield A-C. + + wiseman shares a cohort with su, and su shares one with cravatt, but wiseman and + cravatt share none. cravatt's post must be absent from wiseman's prompt even though + su can see it — and wiseman is given SU_PROFILE so scientific relevance cannot be + the thing doing the filtering. + + Control: su, who does share a cohort with cravatt, selects the same post. Without + that leg, wiseman's non-selection is equally explained by a model that selects + nothing. + """ + from src.services.cohorts import compute_gates + + c1, c2 = uuid.uuid4(), uuid.uuid4() + gates, reason = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman"), (c2, "su"), (c2, "cravatt")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy="isolated", cohort_count=2, + ) + assert reason is None + assert "cravatt" in gates["su"] and "wiseman" in gates["su"], gates["su"] + assert "cravatt" not in gates["wiseman"], ( + f"precondition: the hub must not be transitive. wiseman gate={gates['wiseman']}" + ) + + w = _agent("wiseman", "WisemanBot") + w._public_profile = SU_PROFILE + seen = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="wiseman", + allowed_sender_ids=gates["wiseman"], + ) + sysp, msgs = w.build_phase2_scan_prompt(_post_dicts(seen)) + assert POST_RELEVANT[:40] not in sysp + str(msgs), "the spoke's prompt leaked it" + sel = _selected_ids(await _call(sysp, msgs)) + assert sel is not None + assert "1000.0002" not in sel + + # Control: the hub does select it. + s = _profiled_agent() + seen_su = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=gates["su"], + ) + assert POST_RELEVANT[:40] in str(_post_dicts(seen_su)) + sel_su = _selected_ids(await _call(*s.build_phase2_scan_prompt(_post_dicts(seen_su)))) + assert sel_su is not None + assert "1000.0002" in sel_su, ( + f"control leg failed: the hub did not select the post either ({sel_su}), so " + "the spoke's non-selection proves nothing" + ) diff --git a/tests/integration/test_cohort_scenarios.py b/tests/integration/test_cohort_scenarios.py new file mode 100644 index 0000000..f6d1203 --- /dev/null +++ b/tests/integration/test_cohort_scenarios.py @@ -0,0 +1,691 @@ +"""Real multi-turn cohort scenarios. Marked real_llm — skipped without an API key. + +Everything else in the cohort suite asserts on logic: given these rows, the gate +computes this. These four assert on an **emergent** outcome — who actually ends up +holding a threaded conversation with whom after real Opus/Sonnet turns — which is not +derivable from the gate computation and is the thing the feature is actually for. + +Two design decisions are load-bearing, and both come from a run that proved nothing: + +1. **Every lab is complementary to every other.** An earlier version gave each cohort + internally-matching interests and made the cohorts mutually irrelevant. The gate-OFF + baseline then produced zero cross-cohort threads for reasons of scientific relevance, + so the gate-ON result was unfalsifiable. Here all four labs are facets of one + problem, so every pair is a plausible collaboration and the gate is the only thing + that can prevent one. + +2. **The roster is trimmed to the agents under test.** With four agents over a dozen + turns each agent gets two or three turns, which is not enough for a *specific* pair + to form a thread — outcome claims came back inconclusive rather than confirmed. + +A third: messages the harness itself posts (the lab introductions, and the opening +messages in a private channel) are recorded and excluded from every pair measurement. +Counting them would make "these two agents conversed" true by construction — the +harness put both of them in the channel. +""" + +import os +import time +import uuid +from dataclasses import dataclass, field + +import pytest +from sqlalchemy import delete, text +from sqlalchemy.ext.asyncio import async_sessionmaker + +import src.agent.simulation as sim +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.transport import NullTransport +from src.config import get_settings as real_settings +from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + Cohort, + CohortAuditEvent, + CohortMembership, + SimulationRun, + User, +) +from src.visibility import VISIBILITY_COLLAB_PRIVATE + +pytestmark = [ + pytest.mark.integration, + pytest.mark.real_llm, + pytest.mark.skipif( + not os.environ.get("ANTHROPIC_API_KEY"), + reason="no ANTHROPIC_API_KEY — scenario runs are opt-in and cost money", + ), +] + +LABS = { + "su": ( + "SuBot", + "genome-scale CRISPR screens mapping E3 ligases to substrates; we need " + "degrader chemistry, imaging of substrate loss, and ternary-complex modelling", + ), + "cravatt": ( + "CravattBot", + "covalent chemoproteomics finding ligandable cysteines on E3 ligases and " + "building degraders; we need screen hits, imaging readouts, and structural " + "modelling", + ), + "wiseman": ( + "WisemanBot", + "quantitative single-cell imaging of substrate degradation kinetics; we need " + "screen hits to watch, degrader chemistry to perturb, and analysis pipelines", + ), + "lotz": ( + "LotzBot", + "ternary-complex modelling and image-analysis pipelines for degradation " + "kinetics; we need screen hits, degrader chemistry, and imaging series", + ), +} + +# 8 turns is enough now that scenarios start from an open thread: Phase 4 fires on +# the first turn rather than waiting for Phase 5 to spontaneously choose to reply. +TURNS = int(os.environ.get("SCENARIO_TURNS", "8")) +BUDGET = int(os.environ.get("SCENARIO_BUDGET", "10")) + + +@dataclass +class ScenarioResult: + """What a scenario run produced. Pairs are always ``(a, b)`` with ``a < b``.""" + + gates: dict = field(default_factory=dict) + # Loose: both agents appear in the same public thread and at least one message in + # it is agent-authored. Used for LEAK assertions, where over-detection is safe. + public_pairs: set = field(default_factory=set) + # Strict: both agents authored a message in the same public thread during a turn. + public_exchanges: set = field(default_factory=set) + private_pairs: set = field(default_factory=set) + # Captured AT the mid-run recompute, not at the end. A concluded thread is popped + # out of active_threads (_close_thread), so reading this at the end of a run + # reports [] for a thread that was correctly grandfathered and then finished — + # which is the successful outcome, misread as the failure. + grandfathered: list = field(default_factory=list) + grandfathered_at_end: list = field(default_factory=list) + # {agent_id: sorted senders whose posts this agent's GATED Phase 2 scan accepted}. + # Accumulated every turn, because interesting_posts is consumed as threads form. + interesting_senders: dict = field(default_factory=dict) + strips: dict = field(default_factory=dict) + messages: int = 0 + agent_messages: int = 0 + turns_taken: int = 0 + errors: list = field(default_factory=list) + # {(root_agent, replier): root_ts} for threads the harness opened as preconditions. + seeded_threads: dict = field(default_factory=dict) + # Diagnostics, so a zero-pair result says WHY rather than just failing. + threads: dict = field(default_factory=dict) + posts_by_agent: dict = field(default_factory=dict) + + def authored_in(self, pair) -> list: + """Who posted into a seeded thread during a turn (seeds excluded).""" + ts = self.seeded_threads.get(tuple(pair)) + if ts is None: + return [] + return self.threads.get(ts, {}).get("authored", []) + + def diagnosis(self) -> str: + return ( + f"turns={self.turns_taken} agent_msgs={self.agent_messages} " + f"by_agent={self.posts_by_agent} interesting={self.interesting_senders} " + f"gf_at_split={self.grandfathered} gf_at_end={self.grandfathered_at_end} " + f"threads={self.threads} " + f"loose={sorted(self.public_pairs)} strict={sorted(self.public_exchanges)} " + f"private={sorted(self.private_pairs)} errors={self.errors}" + ) + + +@pytest.fixture +async def scenario_db(engine): + """A committing factory plus cleanup. + + Deliberately not the rolled-back ``db_session``: the engine opens its own sessions + and commits, and that is the path under test. + """ + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + # _build_engine rebinds these module globals directly rather than via monkeypatch, + # because the engine reads them from a dozen call sites during a real turn. Restore + # them here: leaving them patched would silently reconfigure every test that runs + # after a scenario in the same session. + saved = { + name: getattr(sim, name) + for name in ("get_settings", "_UNIVERSAL_CHANNELS", "_CHANNEL_KEYWORDS") + } + try: + yield factory, run_id + finally: + for name, value in saved.items(): + setattr(sim, name, value) + # In a finally too: a failing scenario would otherwise leave its roster and + # cohorts behind for every later test to trip over. + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute( + delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id) + ) + await db.execute( + delete(AgentChannel).where(AgentChannel.simulation_run_id == run_id) + ) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute( + delete(AgentRegistry).where(AgentRegistry.agent_id.in_(tuple(LABS))) + ) + await db.execute(delete(User).where(User.email.like("%@scen.test"))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +async def _seed_roster(factory, run_id, roster): + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for i, aid in enumerate(roster): + u = User( + id=uuid.uuid4(), + orcid=f"9999-0000-0009-{i:04d}", + email=f"{aid}@scen.test", + name=f"PI {aid}", + onboarding_complete=True, + access_status="allowed", + ) + db.add(u) + await db.flush() + db.add(AgentRegistry( + agent_id=aid, bot_name=LABS[aid][0], pi_name=f"PI {aid}", + user_id=u.id, status="active", + )) + await db.commit() + + +async def _set_topology(factory, mapping): + async with factory() as db: + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + for name, members in (mapping or {}).items(): + c = Cohort(name=name) + db.add(c) + await db.flush() + for aid in members: + db.add(CohortMembership(cohort_id=c.id, agent_id=aid)) + await db.commit() + + +async def _all_message_ts(factory, run_id) -> set[str]: + async with factory() as db: + rows = (await db.execute( + text("select message_ts from agent_messages where simulation_run_id = :r"), + {"r": run_id}, + )).all() + return {r[0] for r in rows} + + +async def _public_threads(factory, run_id, exclude_ts): + """``{thread_key: {"all": {agent_ids}, "authored": {agent_ids}}}`` for public bots. + + The thread key is ``coalesce(thread_ts, message_ts)``, so a root and its replies + share one key. ``authored`` excludes the harness's own seeded messages, which is + what keeps "these two conversed" from being true by construction. + """ + async with factory() as db: + rows = (await db.execute(text(""" + select agent_id, coalesce(thread_ts, message_ts) as k, message_ts, + thread_ts is not null as is_reply + from agent_messages + where simulation_run_id = :r and is_bot and visibility = 'public' + and agent_id is not null + """), {"r": run_id})).all() + out: dict[str, dict] = {} + for aid, key, mts, is_reply in rows: + slot = out.setdefault(key, {"all": set(), "authored": set(), "replies": 0}) + slot["all"].add(aid) + if mts not in exclude_ts: + slot["authored"].add(aid) + if is_reply: + slot["replies"] += 1 + return out + + +def _pairs_from_threads(threads, *, strict): + """Pairs co-present in a thread. + + ``strict`` requires both agents to have authored a message in it during a turn. + Loose requires both to be present with at least one authored message in the thread + from either side — that is, a real interaction happened there, even if one-sided. + """ + out = set() + for slot in threads.values(): + who = slot["authored"] if strict else slot["all"] + if not strict and not slot["authored"]: + continue + members = sorted(who) + for i, a in enumerate(members): + for b in members[i + 1:]: + out.add((a, b)) + return out + + +async def _private_channel_pairs(factory, run_id, exclude_ts): + """Pairs that both posted into the same collab_private channel *during turns*. + + Private channels are flat — the agents post messages, not threaded replies — so + co-presence is the signal. The harness's own opening messages are excluded, without + which the pair would be true by construction. + """ + async with factory() as db: + rows = (await db.execute(text(""" + select agent_id, channel_name, message_ts from agent_messages + where simulation_run_id = :r and is_bot + and visibility = 'collab_private' + """), {"r": run_id})).all() + by_channel: dict[str, set[str]] = {} + for aid, ch, mts in rows: + if mts in exclude_ts or aid is None: + continue + by_channel.setdefault(ch, set()).add(aid) + out = set() + for who in by_channel.values(): + for a in sorted(who): + for b in sorted(who): + if a < b: + out.add((a, b)) + return out + + +def _build_engine(factory, run_id, roster, policy): + # ONE public channel for the whole scenario. + # + # Phase 1 joins channels by keyword-matching the profile, and Phase 5 posts into + # whichever of the agent's subscribed channels the model names. With the real + # seven-channel workspace the agents scattered: in a measured 8-turn run su joined + # {general, aging-and-longevity, funding-opportunities} and cravatt joined all + # seven, then posted into #chemical-biology and #drug-repurposing — channels su does + # not read. Neither agent ever saw enough of the other to open a thread, and every + # outcome claim came back INCONCLUSIVE. + # + # Collapsing the workspace to #general is a property of the test *environment*, not + # of the behaviour under test: it makes the agents co-present, which is the + # precondition for the gate to be the deciding factor in whether they converse. The + # gate itself, the read paths and the topology are all untouched. + sim._UNIVERSAL_CHANNELS = {"general"} + sim._CHANNEL_KEYWORDS = {} + + patched = real_settings().model_copy(update={ + "cohort_isolation_enabled": True, + "cohort_default_policy": policy, + "max_consecutive_reactive_turns": 3, + "turn_delay_seconds": 0.0, + "phase5_skip_probability": 0.0, + }) + sim.get_settings = lambda: patched + + agents = [] + for aid in roster: + bot, summary = LABS[aid] + a = Agent(agent_id=aid, bot_name=bot, pi_name=f"PI {aid}") + # The cached-profile seam: a real profile without touching disk or the DB. + a._public_profile = f"# {aid.capitalize()} Lab\n\n{summary}\n" + a._private_profile = "No private instructions yet." + agents.append(a) + + eng = SimulationEngine( + agents=agents, + slack_clients={a: NullTransport(a) for a in roster}, + budget_cap=BUDGET, + session_factory=factory, + simulation_run_id=run_id, + slack_enabled=False, + ) + eng.message_log.set_bot_name_map({LABS[a][0].lower(): a for a in roster}) + eng._bot_name_to_id = {LABS[a][0].lower(): a for a in roster} + eng.message_log.set_persist_callback(eng._enqueue_persist) + # Populates _channel_id_map and _channel_visibility for the seeded channels. + # Resets _channel_visibility wholesale, so any private channel must be registered + # after this call, not before. + eng._ensure_seeded_channels() + return eng + + +async def _seed_thread(factory, eng, run_id, root_agent, replier): + """Open a thread between two agents and register it on both, as a resumed run does. + + The thread's *existence* is a precondition here, not the claim. Left to chance it is + an unreliable one: measured over 16 real turns with two agents, Phase 5 chose "skip" + or "new post" almost every time and produced 3 agent messages and zero threaded + replies. Waiting for a specific pair to spontaneously thread up made every downstream + outcome claim INCONCLUSIVE rather than wrong. + + This mirrors `_rebuild_agent_state`, which is what happens on every resumed run: the + thread exists in the log and both agents carry a ThreadState for it. What the + scenario then measures is emergent — whether a real model continues the thread, and + whether the gate marks or blocks it. See v2 §8, which frames the resumed-run rebuild + as the normal path rather than an edge case. + + Returns the thread's root ts. Both messages are seeds and are excluded from every + pair measurement. + """ + from src.agent.state import ThreadState + + await eng._post_message( + root_agent, "general", + f"Concretely: {LABS[root_agent][1]}. Proposing a first joint experiment — what " + "would you need from us to make it work?", + ) + await eng._flush_persisted() + root_ts = max( + e.ts for e in eng.message_log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id=replier, + allowed_sender_ids=None, + ) if e.sender_agent_id == root_agent + ) + await eng._post_message( + replier, "general", + f"Interested. On our side: {LABS[replier][1]}. What is the smallest pilot that " + "would tell us whether this works?", + thread_ts=root_ts, + ) + await eng._flush_persisted() + + for owner, other in ((root_agent, replier), (replier, root_agent)): + eng.agents[owner].state.active_threads[root_ts] = ThreadState( + thread_id=root_ts, channel="general", other_agent_id=other, + message_count=2, has_pending_reply=(owner == root_agent), + ) + return root_ts + + +async def run_scenario( + factory, run_id, *, policy, topology, roster, + turns=TURNS, private_pair=None, mid_run=None, seed_threads=(), +): + """Drive real turns and return the emergent outcome. + + ``mid_run`` is ``(turn_index, new_topology)`` — applied before that turn and + followed by a gate recompute, which is how grandfathering gets triggered. + + ``seed_threads`` is a sequence of ``(root_agent, replier)`` pairs; each opens a + thread as a resumed run would (see ``_seed_thread``). + """ + await _seed_roster(factory, run_id, roster) + await _set_topology(factory, topology) + eng = _build_engine(factory, run_id, roster, policy) + await eng._recompute_allowed_sender_ids() + gates = { + a: (None if x.allowed_sender_ids is None else set(x.allowed_sender_ids)) + for a, x in eng.agents.items() + } + + if private_pair: + a, b = private_pair + name = f"collab-priv-{a}-{b}" + async with factory() as db: + db.add(AgentChannel( + simulation_run_id=run_id, channel_id=f"local:{name}", + channel_name=name, channel_type="collaboration", + visibility=VISIBILITY_COLLAB_PRIVATE, created_by_agent=a, + )) + await db.commit() + eng._channel_visibility[name] = VISIBILITY_COLLAB_PRIVATE + eng._channel_id_map[name] = f"local:{name}" + for aid in (a, b): + eng.agents[aid].state.subscribed_channels.add(name) + await eng._post_message( + aid, name, + f"(private refinement channel) {LABS[aid][1]} — what would a concrete " + f"first experiment between our labs look like?", + ) + + for aid in roster: + await eng._post_message( + aid, "general", + f"Introducing our lab: {LABS[aid][1]}. Keen to hear from complementary " + "groups.", + ) + await eng._flush_persisted() + + seeded_thread_ids = {} + for root_agent, replier in seed_threads: + seeded_thread_ids[(root_agent, replier)] = await _seed_thread( + factory, eng, run_id, root_agent, replier + ) + + # Everything written so far is the harness's, not the agents'. Excluded from every + # pair measurement below. + seed_ts = await _all_message_ts(factory, run_id) + + def _grandfathered_now(): + return sorted( + (a, th.thread_id) + for a, x in eng.agents.items() + for th in x.state.active_threads.values() + if th.grandfathered + ) + + errors = [] + taken = 0 + grandfathered_at_split = [] + interesting = {a: set() for a in roster} + for t in range(turns): + if mid_run and t == mid_run[0]: + await _set_topology(factory, mid_run[1]) + await eng._recompute_allowed_sender_ids() + # Snapshot here: by the end of the run a grandfathered thread that did what + # §8 wants — concluded — has been popped out of active_threads. + grandfathered_at_split = _grandfathered_now() + agent = eng._select_agent() + if agent is None: + break + taken += 1 + try: + did = await eng._run_turn(agent) + except Exception as exc: # a turn must never abort the whole scenario + errors.append(f"{agent.agent_id}: {type(exc).__name__}: {exc}") + did = False + agent.state.last_selected = time.time() + eng._last_llm_caller = agent.agent_id if did else None + # Phase 2's output is consumed as threads form, so accumulate per turn. + for aid, a in eng.agents.items(): + interesting[aid].update( + p.sender_agent_id for p in a.state.interesting_posts + if p.sender_agent_id + ) + await eng._flush_persisted() + + await eng._flush_persisted() + async with factory() as db: + total = (await db.execute(text( + "select count(*) from agent_messages where simulation_run_id = :r" + ), {"r": run_id})).scalar() + by_agent = dict((await db.execute(text(""" + select agent_id, count(*) from agent_messages + where simulation_run_id = :r and is_bot and agent_id is not null + and message_ts not in (select unnest(cast(:seeds as text[]))) + group by agent_id + """), {"r": run_id, "seeds": list(seed_ts)})).all()) + + threads = await _public_threads(factory, run_id, seed_ts) + return ScenarioResult( + gates=gates, + public_pairs=_pairs_from_threads(threads, strict=False), + public_exchanges=_pairs_from_threads(threads, strict=True), + private_pairs=await _private_channel_pairs(factory, run_id, seed_ts), + threads={ + k: {"all": sorted(v["all"]), "authored": sorted(v["authored"]), + "replies": v["replies"]} + for k, v in threads.items() + }, + posts_by_agent=by_agent, + grandfathered=grandfathered_at_split, + grandfathered_at_end=_grandfathered_now(), + interesting_senders={a: sorted(v) for a, v in interesting.items()}, + seeded_threads=seeded_thread_ids, + strips=dict(eng._cohort_tags_stripped), + messages=total, + agent_messages=total - len(seed_ts), + turns_taken=taken, + errors=errors, + ) + + +async def test_harness_produces_conversation_at_all(scenario_db): + """Self-test, and the positive control the other four rest on. + + A permissive single-cohort run with one open thread must produce at least one + agent-authored message in that thread. If it does not, every absence assertion in + this module is worthless, and this is the test that tells you so — run it first + whenever a scenario comes back empty. + """ + factory, run_id = scenario_db + res = await run_scenario( + factory, run_id, policy="isolated", + topology={"alpha": ["su", "cravatt"]}, roster=["su", "cravatt"], + seed_threads=[("su", "cravatt")], + ) + assert not res.errors, res.errors + assert res.agent_messages >= 1, ( + f"no agent authored anything in {res.turns_taken} turns. {res.diagnosis()}" + ) + assert res.authored_in(("su", "cravatt")), ( + "no real model replied into an OPEN thread between two cohort-mates, so no " + f"scenario built on this harness can prove anything. {res.diagnosis()}" + ) + assert ("cravatt", "su") in res.public_pairs, res.diagnosis() + + +async def test_open_policy_lets_an_uncohorted_agent_be_acted_on(scenario_db): + """§5.2 end to end, measured on the read path the gate actually filters. + + Before the asymmetry fix, `su`'s gate was `{su, wiseman}` — it excluded the + uncohorted agent, so cravatt's posts never reached su's Phase 2 scan and su could + never engage. cravatt could react to anyone and be answered by nobody. + + The assertion is that su's **gated** scan accepted a post authored by cravatt. That + is exactly what the bug prevented, and unlike waiting for a thread to spontaneously + form it happens on the first turn su takes. + + Control: cravatt, whose gate is off entirely, must likewise find su's posts + interesting. If neither direction fired, the run produced nothing and the result is + inconclusive rather than a pass. + """ + factory, run_id = scenario_db + res = await run_scenario( + factory, run_id, policy="open", + topology={"alpha": ["su", "wiseman"]}, roster=["su", "cravatt"], + ) + assert not res.errors, res.errors + assert res.gates["cravatt"] is None, "the uncohorted agent must be unrestricted" + assert "cravatt" in res.gates["su"], ( + f"the open-policy fix is not in effect. su gate={res.gates['su']}" + ) + assert "su" in res.interesting_senders["cravatt"], ( + f"INCONCLUSIVE: the unrestricted agent found nothing interesting, so the gated " + f"direction below proves nothing. {res.diagnosis()}" + ) + assert "cravatt" in res.interesting_senders["su"], ( + "REGRESSED: a cohorted agent's gated scan rejected the uncohorted agent's " + f"posts under policy=open. {res.diagnosis()}" + ) + + +async def test_hub_converses_with_both_sides_but_spokes_do_not(scenario_db): + """Non-transitivity under real conversation. + + su is in both cohorts; cravatt and wiseman share none. One thread is opened on each + of su's two legs as a precondition. The claims are emergent: a real model keeps at + least one of them alive, and no thread ever forms between the two spokes. + + The presence leg is the control — a run where nothing was said at all would satisfy + the leak assertion by itself. + """ + factory, run_id = scenario_db + res = await run_scenario( + factory, run_id, policy="isolated", + topology={"alpha": ["su", "wiseman"], "beta": ["su", "cravatt"]}, + roster=["su", "cravatt", "wiseman"], + seed_threads=[("su", "wiseman"), ("cravatt", "su")], + ) + assert not res.errors, res.errors + assert res.gates["su"] == {"su", "cravatt", "wiseman"} + assert "cravatt" not in res.gates["wiseman"] + assert "wiseman" not in res.gates["cravatt"] + + assert res.authored_in(("su", "wiseman")) or res.authored_in(("cravatt", "su")), ( + f"INCONCLUSIVE: the hub said nothing on either leg. {res.diagnosis()}" + ) + assert ("cravatt", "wiseman") not in res.public_pairs, ( + f"LEAK: two spokes sharing no cohort ended up in one thread. {res.diagnosis()}" + ) + # And neither spoke posted into the other spoke's thread with the hub. + assert "cravatt" not in res.authored_in(("su", "wiseman")), res.diagnosis() + assert "wiseman" not in res.authored_in(("cravatt", "su")), res.diagnosis() + + +async def test_grandfathered_thread_survives_a_mid_run_split(scenario_db): + """§8 under real conversational load. + + A thread is open between two cohort-mates; the topology then splits them. Three + things must hold, and the third is the one a marked-but-stalled thread would fail: + + 1. the engine marks the thread grandfathered on the recompute; + 2. it loses reactive priority (asserted deterministically in the live suite); + 3. a **real model** still writes into it, so the conversation can conclude. + """ + factory, run_id = scenario_db + res = await run_scenario( + factory, run_id, policy="isolated", + topology={"alpha": ["su", "cravatt"]}, roster=["su", "cravatt"], + seed_threads=[("su", "cravatt")], + # Split before the first turn. The thread was opened while they were mates (the + # topology above), so the precondition holds; splitting later is a race — at + # turn 4 the thread had already concluded and been popped out of + # active_threads, leaving nothing to mark. Every authored message below is + # therefore post-split, which is what makes the third assertion mean something. + mid_run=(0, {"alpha": ["su"], "beta": ["cravatt"]}), + ) + assert not res.errors, res.errors + assert res.gates["su"] == {"su", "cravatt"}, "the gate BEFORE the split" + root_ts = res.seeded_threads[("su", "cravatt")] + assert res.grandfathered, ( + "the open cross-cohort thread was not marked when the topology split. " + f"{res.diagnosis()}" + ) + assert {t for _, t in res.grandfathered} == {root_ts}, ( + f"the wrong thread was grandfathered. {res.diagnosis()}" + ) + assert sorted(a for a, _ in res.grandfathered) == ["cravatt", "su"], ( + f"both sides of the thread must be marked, not just one. {res.diagnosis()}" + ) + assert res.authored_in(("su", "cravatt")), ( + "the grandfathered thread received nothing at all — it stalled instead of " + f"concluding. {res.diagnosis()}" + ) + + +async def test_private_channel_beats_the_cohort_gate(scenario_db): + """§7: a PI-created pairing outranks an admin grouping. + + su and cravatt are in different cohorts and maximally gated — each can act only on + itself. They must still converse in the channel the PI made for them. + + Control: they must NOT converse in the public channel. Without that leg the private + result is equally explained by the gate not being in force at all. + """ + factory, run_id = scenario_db + res = await run_scenario( + factory, run_id, policy="isolated", + topology={"alpha": ["su"], "beta": ["cravatt"]}, roster=["su", "cravatt"], + private_pair=("su", "cravatt"), + ) + assert not res.errors, res.errors + assert res.gates["su"] == {"su"} and res.gates["cravatt"] == {"cravatt"} + assert ("cravatt", "su") in res.private_pairs, ( + "INCONCLUSIVE OR REGRESSED: the two agents did not both post into the channel " + f"the PI created for them, during a turn. {res.diagnosis()}" + ) + assert ("cravatt", "su") not in res.public_pairs, ( + f"control leg failed: they also conversed publicly, so the gate is off. " + f"{res.diagnosis()}" + ) diff --git a/tests/integration/test_conversation_feed.py b/tests/integration/test_conversation_feed.py new file mode 100644 index 0000000..e36c2b2 --- /dev/null +++ b/tests/integration/test_conversation_feed.py @@ -0,0 +1,926 @@ +"""The conversations feed's visibility gate, and its parity with the engine. + +The page must show exactly what the viewing agent's bot is allowed to act on. +The engine decides that in memory (``_entry_allowed``); the page decides it in +SQL (``gate_clause``). Two implementations of one rule is a drift hazard, so the +parity test below drives BOTH from the engine's own ``DECISION_TABLE``. +""" + +import pytest +from sqlalchemy import select + +from src.agent.message_log import _entry_allowed +from src.models import AgentMessage, Cohort, CohortMembership +from src.services.conversation_feed import gate_clause, resolve_agent_gate +from tests import factories +from tests.integration.test_agent_page import _auth +from tests.unit.test_cohort_isolation import DECISION_TABLE, _post + +pytestmark = pytest.mark.integration + + +async def _cohort(db, name, *agent_ids): + c = Cohort(name=name) + db.add(c) + await db.flush() + for aid in agent_ids: + db.add(CohortMembership(cohort_id=c.id, agent_id=aid)) + await db.flush() + return c + + +@pytest.mark.parametrize( + "name,kwargs,gate,expected", DECISION_TABLE, ids=[r[0] for r in DECISION_TABLE] +) +async def test_gate_clause_matches_entry_allowed( + db_session, name, kwargs, gate, expected +): + """Every row of the engine's §5.1 table, decided by SQL instead of Python.""" + run = await factories.make_simulation_run(db_session) + row_kwargs = dict(agent_id="x", is_bot=True, visibility="public") + row_kwargs.update( + {k: v for k, v in kwargs.items() if k in ("agent_id", "is_bot", "visibility")} + ) + msg = await factories.make_agent_message( + db_session, run=run, message_ts="1.0001", content="body", **row_kwargs + ) + await db_session.flush() + + found = (await db_session.execute( + select(AgentMessage.id).where( + AgentMessage.simulation_run_id == run.id, + gate_clause(gate), + ) + )).scalars().all() + sql_visible = msg.id in found + + entry_kwargs = dict(ts="1", channel="c", agent_id="x", name="X", content="") + entry_kwargs.update(kwargs) + python_visible = _entry_allowed(_post(**entry_kwargs), gate) + + assert sql_visible == expected, f"SQL disagreed with the table on: {name}" + assert sql_visible == python_visible, ( + f"gate_clause and _entry_allowed disagree on: {name}" + ) + + +async def test_gate_is_the_union_of_co_members(db_session, monkeypatch): + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent(db_session, agent_id="spoke1", bot_name="Spoke1Bot") + await factories.make_agent(db_session, agent_id="spoke2", bot_name="Spoke2Bot") + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "pair1", "spoke1", "hub") + await _cohort(db_session, "pair2", "spoke2", "hub") + + assert await resolve_agent_gate(db_session, "spoke1") == {"spoke1", "hub"} + assert await resolve_agent_gate(db_session, "spoke2") == {"spoke2", "hub"} + assert await resolve_agent_gate(db_session, "hub") == {"spoke1", "spoke2", "hub"} + + +async def test_uncohorted_agent_is_isolated_under_policy_isolated( + db_session, monkeypatch +): + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent(db_session, agent_id="lonely", bot_name="LonelyBot") + await factories.make_agent(db_session, agent_id="other", bot_name="OtherBot") + await _cohort(db_session, "somepair", "other") + + assert await resolve_agent_gate(db_session, "lonely") == set() + + +async def test_gate_is_off_when_isolation_is_disabled(db_session, monkeypatch): + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", False, raising=False) + + await factories.make_agent(db_session, agent_id="anyone", bot_name="AnyoneBot") + + assert await resolve_agent_gate(db_session, "anyone") is None + + +async def test_an_inactive_viewing_agent_still_resolves(db_session, monkeypatch): + """compute_gates only keys the roster it is given, and the conversations route + admits status 'inactive'. Without adding the viewer to the roster, 'sleeper' + would be absent from compute_gates' agent_ids, so gates.get('sleeper') would + silently return None (gate off / unrestricted) instead of the viewer's real + cohort gate {'sleeper', 'awake'} — the opposite of the intended isolation, + and worse than a KeyError because it fails open rather than loud.""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent( + db_session, agent_id="sleeper", bot_name="SleeperBot", status="inactive" + ) + await factories.make_agent(db_session, agent_id="awake", bot_name="AwakeBot") + await _cohort(db_session, "mixed", "sleeper", "awake") + + assert await resolve_agent_gate(db_session, "sleeper") == {"sleeper", "awake"} + + +async def test_preflight_refusal_fails_open_loudly(db_session, monkeypatch, caplog): + """Under policy="isolated" with zero live memberships, compute_gates' preflight + forces isolation OFF for every agent (src/services/cohorts.py's roster-wide- + silence guard) — resolve_agent_gate must still return that `None` (this is + deliberate engine fail-open behaviour, not a bug to "fix" here), but it must + say so loudly: an admin can reach exactly this state with one click ("all / + none" on every column of /admin/cohorts/topology, then save), and before this + fix nothing — no log line, no banner — recorded that every PI's conversations + feed had gone fully ungated as a result.""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + await factories.make_agent(db_session, agent_id="solo", bot_name="SoloBot") + # Deliberately zero cohorts/memberships — the roster-wide-silence state. + + with caplog.at_level("WARNING"): + gate = await resolve_agent_gate(db_session, "solo") + + assert gate is None, ( + "the accepted fail-open behaviour: do not change this, only make it loud" + ) + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert any("UNGATED" in r.getMessage() for r in warnings), ( + f"expected a warning naming the preflight refusal and its consequence, " + f"got: {[r.getMessage() for r in warnings]}" + ) + assert any("solo" in r.getMessage() for r in warnings) + + +async def test_no_warning_when_gate_is_simply_off(db_session, monkeypatch, caplog): + """Control for the test above: isolation disabled is the ordinary, silent + gate-off path (§5.1's first row) — it must not trip the preflight-refusal + warning, or every request would log at WARNING and the signal would be + worthless.""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", False, raising=False) + + await factories.make_agent(db_session, agent_id="quiet", bot_name="QuietBot") + + with caplog.at_level("WARNING"): + gate = await resolve_agent_gate(db_session, "quiet") + + assert gate is None + assert not any("UNGATED" in r.getMessage() for r in caplog.records) + + +async def test_a_spoke_pi_does_not_see_another_spokes_bot( + client, db_session, monkeypatch +): + """The star topology: two spokes and a hub. Spoke 1's PI must not see + Spoke 2's bot, and MUST still see the hub (the positive control).""" + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi1 = await factories.make_user(db_session, name="Spoke One", email="s1@example.org") + await factories.make_agent( + db_session, user=pi1, agent_id="spoke1", bot_name="Spoke1Bot", pi_name="Spoke One" + ) + await factories.make_agent(db_session, agent_id="spoke2", bot_name="Spoke2Bot") + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "pair1", "spoke1", "hub") + await _cohort(db_session, "pair2", "spoke2", "hub") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + # Spoke 1's own post is what puts #general in its channel set. + await factories.make_agent_message( + db_session, agent_id="spoke1", message_ts="1.0001", + content="MINE-own-post", sender_name="Spoke1Bot", **common + ) + await factories.make_agent_message( + db_session, agent_id="hub", message_ts="1.0002", + content="HUB-visible-post", sender_name="HubBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="spoke2", message_ts="1.0003", + content="LEAK-other-spoke-post", sender_name="Spoke2Bot", **common + ) + await db_session.commit() + + page = await client.get("/agent/spoke1/conversations", headers=_auth(pi1.id)) + assert page.status_code == 200 + assert "MINE-own-post" in page.text + assert "HUB-visible-post" in page.text, "positive control: the hub must be visible" + assert "LEAK-other-spoke-post" not in page.text + assert "Spoke2Bot" not in page.text + + +async def test_a_pi_message_still_renders_under_the_gate( + client, db_session, monkeypatch +): + """is_bot=False bypasses the gate — the human bypass must survive. + + The gate must be genuinely ON here, or this proves nothing: with zero + cohorts defined, compute_gates' preflight refuses under + policy="isolated" (roster-wide-silence guard) and returns gate=None for + every agent, which makes gate_clause a no-op regardless of is_bot. Putting + "solo" in a cohort of its own makes resolve_agent_gate return a real, + non-None set, so the human row can only pass through the is_bot bypass + branch of gate_clause, not through the gate being off — asserted below. + """ + from src.config import get_settings + from src.services.conversation_feed import resolve_agent_gate + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Solo PI", email="solo@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="solo", bot_name="SoloBot", pi_name="Solo PI" + ) + await _cohort(db_session, "solo-cohort", "solo") + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="solo", message_ts="2.0001", + content="BOT-anchor", sender_name="SoloBot", **common + ) + await factories.make_agent_message( + db_session, agent_id=None, is_bot=False, message_ts="2.0002", + content="HUMAN-said-this", sender_name="Solo PI (PI)", **common + ) + await db_session.commit() + + assert await resolve_agent_gate(db_session, "solo") == {"solo"}, ( + "the gate must be a real, non-None set here, or the bypass this test " + "targets is never actually exercised" + ) + + page = await client.get("/agent/solo/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "HUMAN-said-this" in page.text + + +async def test_an_uncohorted_agent_still_sees_its_own_posts( + client, db_session, monkeypatch +): + """Under policy="isolated" an active-but-uncohorted agent's gate is the + EMPTY set (not None) — deliberately, so it cannot read any other lab's + traffic. But activation and cohort assignment are separate admin steps + (see CLAUDE.md's onboarding order: Provision -> Approve & Activate happens + before any admin adds the agent to a cohort), so a PI must still see their + OWN bot's posts in that gap, or their page goes blank the moment their bot + goes live. This is the safe, deliberate divergence from `_entry_allowed` + documented at the `own_or_gated` clause in agent_page.py: it can only ever + admit this agent's own rows, never another agent's.""" + from src.config import get_settings + from src.services.conversation_feed import resolve_agent_gate + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Lonely PI", email="lonely@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="lonely", bot_name="LonelyBot", pi_name="Lonely PI" + ) + await factories.make_agent(db_session, agent_id="other", bot_name="OtherBot") + await _cohort(db_session, "other-only", "other") # "lonely" is deliberately left out + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="lonely", message_ts="8.0001", + content="LONELY-OWN-POST", sender_name="LonelyBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="other", message_ts="8.0002", + content="OTHER-LAB-POST", sender_name="OtherBot", **common + ) + await db_session.commit() + + assert await resolve_agent_gate(db_session, "lonely") == set(), ( + "this test targets the isolated-empty-set case specifically" + ) + + page = await client.get("/agent/lonely/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "LONELY-OWN-POST" in page.text, ( + "an uncohorted agent's PI must still see their own bot's posts" + ) + assert "OTHER-LAB-POST" not in page.text + + +async def test_gate_is_applied_before_limit_not_after( + client, db_session, monkeypatch +): + """`_ROOT_LIMIT` must select the top-N GATE-PASSING roots, not the top-N + roots with the gate applied afterward in Python. Flood the channel with 60 + out-of-cohort roots, all newer (higher posted_at) than a single in-cohort + root belonging to a cohort-mate. If the gate ran after `.limit(_ROOT_LIMIT)` + instead of in the SQL WHERE, the initial fetch would already be full of the + 50 newest out-of-cohort rows and the in-cohort root — older than all 60 — + would never be fetched at all, gate or no gate.""" + from src.config import get_settings + from src.routers.agent_page import _ROOT_LIMIT + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + assert _ROOT_LIMIT < 60, "the flood must exceed the window for this test to prove anything" + + pi = await factories.make_user(db_session, name="Flooded PI", email="flooded@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="victim", bot_name="VictimBot", pi_name="Flooded PI" + ) + await factories.make_agent(db_session, agent_id="mate", bot_name="MateBot") + await factories.make_agent(db_session, agent_id="flooder", bot_name="FlooderBot") + await _cohort(db_session, "victim-mate", "victim", "mate") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + for i in range(60): + await factories.make_agent_message( + db_session, agent_id="flooder", message_ts=f"7.{i:04d}", + phase="new_post", content=f"FLOOD-{i}", sender_name="FlooderBot", + posted_at=1000.0 + i, **common + ) + await factories.make_agent_message( + db_session, agent_id="mate", message_ts="7.9999", phase="new_post", + content="SURVIVOR-ROOT", sender_name="MateBot", posted_at=1.0, **common + ) + await db_session.commit() + + page = await client.get("/agent/victim/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "SURVIVOR-ROOT" in page.text, ( + "the in-cohort root, though older than all 60 out-of-cohort floods, must " + "still render — proving the gate ran in SQL before LIMIT, not in Python after" + ) + + +async def test_reply_count_excludes_out_of_cohort_replies( + client, db_session, monkeypatch +): + """`reply_count` must be computed with the SAME gate as the roots query, so + the badge (Task 6) can never promise a reply the thread-expand endpoint + (Task 5) will not show. A root with one in-cohort reply and one + out-of-cohort reply must report reply_count == 1, not 2. + + The template does not render reply_count yet (Task 6 owns that), so this + intercepts the context handed to templates.TemplateResponse rather than + reading it out of rendered HTML. + """ + import src.routers.agent_page as agent_page_module + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Counter PI", email="counter@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="counter", bot_name="CounterBot", pi_name="Counter PI" + ) + await factories.make_agent(db_session, agent_id="mate", bot_name="MateBot") + await factories.make_agent(db_session, agent_id="outsider", bot_name="OutsiderBot") + await _cohort(db_session, "counter-mate", "counter", "mate") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="counter", message_ts="6.0001", phase="new_post", + content="COUNT-ROOT", sender_name="CounterBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="mate", message_ts="6.0002", thread_ts="6.0001", + phase="thread_reply", content="IN-COHORT-REPLY", sender_name="MateBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="outsider", message_ts="6.0003", thread_ts="6.0001", + phase="thread_reply", content="OUT-OF-COHORT-REPLY", sender_name="OutsiderBot", + **common + ) + await db_session.commit() + + captured: dict = {} + original_response = agent_page_module.templates.TemplateResponse + + def _capture(request, name, context, *args, **kwargs): + captured["messages"] = context.get("messages") + return original_response(request, name, context, *args, **kwargs) + + monkeypatch.setattr(agent_page_module.templates, "TemplateResponse", _capture) + + page = await client.get("/agent/counter/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + + roots_by_content = {m["content"]: m for m in captured["messages"]} + assert "COUNT-ROOT" in roots_by_content + assert roots_by_content["COUNT-ROOT"]["reply_count"] == 1, ( + "reply_count must be gated the same as the roots query — it should count " + "only the in-cohort reply, not the out-of-cohort one" + ) + + +async def test_a_reply_from_a_different_channel_sharing_a_thread_ts_is_excluded( + client, db_session, monkeypatch +): + """Latent-bug regression (audit finding F4). `thread_ts` alone is not proof a + row belongs to a given root's conversation — only `uq_agent_messages_run_ts` + (message_ts unique per run) is guaranteed, and nothing enforces that a row's + `thread_ts` names a root in ITS OWN channel. Construct exactly the scenario + the finding describes: a `collab_private` row (which `gate_clause` passes + unconditionally, regardless of cohort) whose `thread_ts` happens to equal a + PUBLIC root's `message_ts`, but which lives in a different channel. Both the + reply count and the thread-expand endpoint must exclude it — it is not a + reply to this root, it only shares its thread_ts by coincidence. + + This state is not currently producible by any real writer (prod has zero + `collab_private` rows), so the row is constructed directly rather than + produced by the app — the point is to pin the read-side defense structurally, + since nothing else in the system rules the coincidence out. + """ + import src.routers.agent_page as agent_page_module + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user( + db_session, name="Channel Bound PI", email="chanbound@example.org" + ) + await factories.make_agent( + db_session, user=pi, agent_id="channelbound", bot_name="ChannelBoundBot", + pi_name="Channel Bound PI", + ) + await factories.make_agent(db_session, agent_id="outsider2", bot_name="Outsider2Bot") + # "channelbound" gets a cohort of its own (not shared with "outsider2") so + # resolve_agent_gate returns a REAL, non-None set — the only reason + # outsider2's row can pass the gate below is the unconditional + # collab_private bypass, not a preflight refusal leaving everything open. + await _cohort(db_session, "channelbound-solo", "channelbound") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="channelbound", message_ts="15.0001", phase="new_post", + channel_name="general", content="REAL-ROOT", sender_name="ChannelBoundBot", + **common, + ) + await factories.make_agent_message( + db_session, agent_id="outsider2", message_ts="15.0002", thread_ts="15.0001", + phase="thread_reply", channel_name="priv-elsewhere", + content="WRONG-CHANNEL-COLLAB-PRIVATE-REPLY", sender_name="Outsider2Bot", + run=run, channel_id="C2", visibility="collab_private", + ) + await db_session.commit() + + assert await resolve_agent_gate(db_session, "channelbound") == {"channelbound"}, ( + "this test targets a REAL, non-None gate — if the gate were off, the " + "collab_private bypass this test targets would never be exercised" + ) + + captured: dict = {} + original_response = agent_page_module.templates.TemplateResponse + + def _capture(request, name, context, *args, **kwargs): + captured["messages"] = context.get("messages") + return original_response(request, name, context, *args, **kwargs) + + monkeypatch.setattr(agent_page_module.templates, "TemplateResponse", _capture) + + page = await client.get("/agent/channelbound/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + roots_by_content = {m["content"]: m for m in captured["messages"]} + assert roots_by_content["REAL-ROOT"]["reply_count"] == 0, ( + "a row in a DIFFERENT channel must not count toward this root's replies " + "merely because thread_ts coincides with the root's message_ts" + ) + + r = await client.get("/agent/channelbound/thread/15.0001", headers=_auth(pi.id)) + assert r.status_code == 200 + assert "WRONG-CHANNEL-COLLAB-PRIVATE-REPLY" not in r.text, ( + "the thread-expand endpoint must not render a different channel's row " + "just because it shares the root's thread_ts" + ) + + +async def test_replies_are_not_listed_as_top_level_rows( + client, db_session, monkeypatch +): + """The feed selects ROOTS. A reply appears via its count, not as its own card.""" + from src.config import get_settings + monkeypatch.setattr( + get_settings(), "cohort_isolation_enabled", False, raising=False + ) + + pi = await factories.make_user(db_session, name="Root PI", email="root@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="rooter", bot_name="RooterBot", pi_name="Root PI" + ) + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="rooter", message_ts="3.0001", phase="new_post", + content="THE-ROOT", sender_name="RooterBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="rooter", message_ts="3.0002", thread_ts="3.0001", + phase="thread_reply", content="THE-REPLY", sender_name="RooterBot", **common + ) + await db_session.commit() + + page = await client.get("/agent/rooter/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "THE-ROOT" in page.text + assert "THE-REPLY" not in page.text, "a reply must not render as a top-level card" + + +async def test_a_delegate_sees_exactly_what_the_owner_sees( + client, db_session, monkeypatch +): + """Access is owner-or-delegate; the gate is the AGENT's, not the viewer's, so + both must get byte-identical feeds.""" + from src.config import get_settings + from src.models import AgentDelegate + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Owner", email="own@example.org") + agent = await factories.make_agent( + db_session, user=pi, agent_id="deleg", bot_name="DelegBot", pi_name="Owner" + ) + await factories.make_agent(db_session, agent_id="stranger", bot_name="StrangerBot") + await _cohort(db_session, "solo", "deleg") + + dee = await factories.make_user(db_session, name="Dee", email="dee2@example.org") + db_session.add(AgentDelegate(agent_registry_id=agent.id, user_id=dee.id)) + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="deleg", message_ts="4.0001", + content="OWN-POST", sender_name="DelegBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="stranger", message_ts="4.0002", + content="OUTSIDER-POST", sender_name="StrangerBot", **common + ) + await db_session.commit() + + owner_page = await client.get("/agent/deleg/conversations", headers=_auth(pi.id)) + dee_page = await client.get("/agent/deleg/conversations", headers=_auth(dee.id)) + assert owner_page.status_code == 200 + assert dee_page.status_code == 200 + assert "OWN-POST" in dee_page.text + assert "OUTSIDER-POST" not in owner_page.text + assert "OUTSIDER-POST" not in dee_page.text + + +# --------------------------------------------------------------------------- +# Task 5: thread expand endpoint (GET /agent/{agent_id}/thread/{message_ts}) +# --------------------------------------------------------------------------- + + +async def _threaded_world(db_session, monkeypatch): + """Spoke1 (owned) + Spoke2 (not owned). + + Spoke1's root (9.0001) has TWO replies: one from cohort-mate `hub` + (HUB-REPLY, in spoke1's gate {spoke1, hub}) and one from `spoke2` + (OUT-OF-COHORT-REPLY, NOT in spoke1's gate — spoke1 and spoke2 are each + paired with `hub` but not with each other). That pairing is deliberate: + it is the only way to prove replies are gated at all, rather than merely + admitted through the root owner's own-post carve-out. Spoke2 additionally + has its own root (9.0003, FOREIGN-ROOT) with no reply, used by the + out-of-cohort-root/IDOR tests below. + """ + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi1 = await factories.make_user(db_session, name="S One", email="t1@example.org") + await factories.make_agent( + db_session, user=pi1, agent_id="spoke1", bot_name="Spoke1Bot", pi_name="S One" + ) + await factories.make_agent(db_session, agent_id="spoke2", bot_name="Spoke2Bot") + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "p1", "spoke1", "hub") + await _cohort(db_session, "p2", "spoke2", "hub") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="spoke1", message_ts="9.0001", phase="new_post", + content="MY-ROOT", sender_name="Spoke1Bot", **common + ) + await factories.make_agent_message( + db_session, agent_id="hub", message_ts="9.0002", thread_ts="9.0001", + phase="thread_reply", content="HUB-REPLY", sender_name="HubBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="spoke2", message_ts="9.0003", phase="new_post", + content="FOREIGN-ROOT", sender_name="Spoke2Bot", **common + ) + await factories.make_agent_message( + db_session, agent_id="spoke2", message_ts="9.0004", thread_ts="9.0001", + phase="thread_reply", content="OUT-OF-COHORT-REPLY", sender_name="Spoke2Bot", + **common + ) + await db_session.commit() + return pi1 + + +async def test_expanding_own_thread_returns_the_gated_replies( + client, db_session, monkeypatch +): + """Positive control (HUB-REPLY, in-cohort) and negative control + (OUT-OF-COHORT-REPLY, from an agent NOT in spoke1's gate) in the same + thread the viewer legitimately owns. This is the exact clause the brief + calls out as the deliberate engine divergence: replies must be gated with + own_or_gated, not merely admitted because the root belongs to the viewer. + Deleting `gated` from the reply_rows query in agent_thread_replies turns + this red (verified by hand — see task-5-report.md). + """ + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/9.0001", headers=_auth(pi1.id)) + assert r.status_code == 200 + assert "HUB-REPLY" in r.text + assert "OUT-OF-COHORT-REPLY" not in r.text, ( + "a reply must be gated even inside a thread the viewer owns the root of" + ) + + +async def test_expanding_an_out_of_cohort_root_is_404(client, db_session, monkeypatch): + """The IDOR guard: message_ts is guessable, so the root must re-pass the gate.""" + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/9.0003", headers=_auth(pi1.id)) + assert r.status_code == 404 + assert "FOREIGN-ROOT" not in r.text + + +async def test_expanding_a_root_from_a_channel_the_viewer_never_posted_in_is_404( + client, db_session, monkeypatch +): + """Authorization check #3 of the four in ``agent_thread_replies``'s + docstring (``agent_page.py:870-874``): the root's channel must be in the + VIEWER's own channel set (``_visible_channels`` — channels this agent has + authored in, plus ``#general``), not merely pass the cohort gate. + + Every other test in this module uses ``channel_name="general"``, which + ``_visible_channels`` adds unconditionally regardless of what the agent has + posted — so ``AgentMessage.channel_name.in_(channels)`` at + ``agent_page.py:907`` is a tautology everywhere else in this file and its + absence would not be caught. This test puts a cohort-mate's root in a + channel the viewer's own agent (``spoke1``) has never authored in + (``secret-room``, not ``general``) so the root passes ``gate_clause`` (the + author, ``hub``, shares a cohort with ``spoke1``) but must still 404 on the + channel-membership check alone. + + Hand-verified: deleting the ``AgentMessage.channel_name.in_(channels)`` + line from the root query in ``agent_thread_replies`` turns this test RED + (200 instead of 404); restoring it turns it GREEN. See + final-fix-report.md for the transcript. + """ + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi1 = await factories.make_user(db_session, name="Chan One", email="chan1@example.org") + await factories.make_agent( + db_session, user=pi1, agent_id="spoke1", bot_name="Spoke1Bot", pi_name="Chan One" + ) + await factories.make_agent(db_session, agent_id="hub", bot_name="HubBot") + await _cohort(db_session, "pair1", "spoke1", "hub") + + run = await factories.make_simulation_run(db_session) + # hub's root lives in a channel spoke1 has never authored in — NOT "general", + # which _visible_channels would add regardless of authorship. + await factories.make_agent_message( + db_session, run=run, agent_id="hub", message_ts="14.0001", phase="new_post", + content="HUB-OTHER-CHANNEL-ROOT", sender_name="HubBot", + channel_name="secret-room", channel_id="C-SECRET", visibility="public", + ) + await db_session.commit() + + from src.services.conversation_feed import resolve_agent_gate + gate = await resolve_agent_gate(db_session, "spoke1") + assert gate == {"spoke1", "hub"}, ( + "this test targets the case where the root PASSES the gate — if the gate " + "itself refused, the 404 below would prove nothing about the channel check" + ) + + r = await client.get("/agent/spoke1/thread/14.0001", headers=_auth(pi1.id)) + assert r.status_code == 404, ( + "a gate-passing root in a channel the viewer never authored in must still " + "404 — the channel-membership check is a separate authorization axis, not " + "implied by the cohort gate" + ) + assert "HUB-OTHER-CHANNEL-ROOT" not in r.text + + +async def test_expanding_a_reply_ts_rather_than_a_root_is_404( + client, db_session, monkeypatch +): + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/9.0002", headers=_auth(pi1.id)) + assert r.status_code == 404 + + +async def test_expanding_an_unknown_ts_is_404(client, db_session, monkeypatch): + pi1 = await _threaded_world(db_session, monkeypatch) + r = await client.get("/agent/spoke1/thread/0.0000", headers=_auth(pi1.id)) + assert r.status_code == 404 + + +async def test_a_stranger_cannot_expand_someone_elses_thread( + client, db_session, monkeypatch +): + await _threaded_world(db_session, monkeypatch) + stranger = await factories.make_user( + db_session, name="Nosy", email="nosy@example.org" + ) + await db_session.commit() + r = await client.get("/agent/spoke1/thread/9.0001", headers=_auth(stranger.id)) + assert r.status_code == 403 + assert "HUB-REPLY" not in r.text + + +async def test_expanding_an_uncohorted_own_thread_is_200_not_404( + client, db_session, monkeypatch +): + """CONTROLLER AMENDMENT case: a PI whose agent is active but uncohorted + (gate == empty set under policy="isolated") must still be able to expand + their OWN thread. A bare `gate_clause(gate)` would resolve `root is None` + here and 404 the PI's own thread — the leaking-inverse of the feed's own + carve-out, which already renders this root and counts this reply in the + badge. `own_or_gated(gate, aid)` must admit both the root and the reply. + """ + from src.config import get_settings + from src.services.conversation_feed import resolve_agent_gate + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user( + db_session, name="Lonely PI", email="lonelyexpand@example.org" + ) + await factories.make_agent( + db_session, user=pi, agent_id="lonely", bot_name="LonelyBot", pi_name="Lonely PI" + ) + await factories.make_agent(db_session, agent_id="other", bot_name="OtherBot") + await _cohort(db_session, "other-only", "other") # "lonely" is deliberately left out + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="lonely", message_ts="11.0001", phase="new_post", + content="LONELY-ROOT", sender_name="LonelyBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="lonely", message_ts="11.0002", thread_ts="11.0001", + phase="thread_reply", content="LONELY-OWN-REPLY", sender_name="LonelyBot", + **common + ) + await db_session.commit() + + assert await resolve_agent_gate(db_session, "lonely") == set(), ( + "this test targets the isolated-empty-set case specifically" + ) + + r = await client.get("/agent/lonely/thread/11.0001", headers=_auth(pi.id)) + assert r.status_code == 200 + assert "LONELY-OWN-REPLY" in r.text + + +# --------------------------------------------------------------------------- +# Task 6: reply badge + expand-on-click +# --------------------------------------------------------------------------- + + +async def test_the_badge_count_equals_the_rendered_reply_count( + client, db_session, monkeypatch +): + """The badge is computed with the same gate as the expansion, so it can never + promise turns the expansion will not show.""" + pi1 = await _threaded_world(db_session, monkeypatch) + + page = await client.get("/agent/spoke1/conversations", headers=_auth(pi1.id)) + assert page.status_code == 200 + assert "1 reply" in page.text + assert "1 replies" not in page.text, "singular/plural must agree with the count" + # The badge text alone doesn't prove the link goes anywhere real — assert + # the anchor for this known root addresses its own thread, not some other + # value (e.g. `m.thread_ts`, which is None on every root). + assert 'href="/agent/spoke1/thread/9.0001"' in page.text + + r = await client.get("/agent/spoke1/thread/9.0001", headers=_auth(pi1.id)) + assert r.status_code == 200 + assert r.text.count("data-reply-row") == 1 + + +async def test_a_root_with_multiple_in_cohort_replies_renders_the_plural_badge( + client, db_session, monkeypatch +): + """`_threaded_world`'s only root has exactly one IN-COHORT reply, by design + (the second reply on that root is deliberately out-of-cohort, to prove + replies are gated at all — see `_threaded_world`'s docstring). So nothing + in this file, before this test, ever produces `reply_count >= 2`: a + template that hardcoded the singular branch (e.g. + `{% if m.reply_count == 1 %}reply{% endif %}` with no `else`, silently + rendering `"Show 2 "` with no noun, or always rendering `"reply"` + regardless of count) would pass every other test here. Built as an + independent root/agent pair rather than adding to `_threaded_world`, + which other tests depend on for an exact reply count of one. + """ + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Plural PI", email="plural@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="plural", bot_name="PluralBot", pi_name="Plural PI" + ) + await factories.make_agent(db_session, agent_id="pluralmate", bot_name="PluralMateBot") + await _cohort(db_session, "plural-mate", "plural", "pluralmate") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="plural", message_ts="12.0001", phase="new_post", + content="PLURAL-ROOT", sender_name="PluralBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="pluralmate", message_ts="12.0002", thread_ts="12.0001", + phase="thread_reply", content="PLURAL-REPLY-1", sender_name="PluralMateBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="pluralmate", message_ts="12.0003", thread_ts="12.0001", + phase="thread_reply", content="PLURAL-REPLY-2", sender_name="PluralMateBot", **common + ) + await db_session.commit() + + page = await client.get("/agent/plural/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert "2 replies" in page.text + assert "2 reply" not in page.text, "singular/plural must agree with the count" + + +async def test_each_roots_href_addresses_its_own_thread_not_a_neighbors( + client, db_session, monkeypatch +): + """Nothing before this test asserted the rendered anchor's `href` value at + all — only that some reply-count text appeared somewhere on the page. A + template bug that interpolated the wrong field (e.g. `m.thread_ts`, which + is None on every root, or a stray reused loop variable that pins every + row's link to the SAME root) would still pass every badge/count test. + Two roots with replies on the same page make a same-value bug visible: + each root's href must be present exactly once, and each must point at its + own `message_ts`, not the other's. + """ + from src.config import get_settings + s = get_settings() + monkeypatch.setattr(s, "cohort_isolation_enabled", True, raising=False) + monkeypatch.setattr(s, "cohort_default_policy", "isolated", raising=False) + + pi = await factories.make_user(db_session, name="Href PI", email="href@example.org") + await factories.make_agent( + db_session, user=pi, agent_id="hreftest", bot_name="HrefBot", pi_name="Href PI" + ) + await factories.make_agent(db_session, agent_id="hrefmate", bot_name="HrefMateBot") + await _cohort(db_session, "href-mate", "hreftest", "hrefmate") + + run = await factories.make_simulation_run(db_session) + common = dict(run=run, channel_name="general", channel_id="C1", visibility="public") + await factories.make_agent_message( + db_session, agent_id="hreftest", message_ts="13.0001", phase="new_post", + content="HREF-ROOT-A", sender_name="HrefBot", posted_at=1.0, **common + ) + await factories.make_agent_message( + db_session, agent_id="hrefmate", message_ts="13.0002", thread_ts="13.0001", + phase="thread_reply", content="HREF-REPLY-A", sender_name="HrefMateBot", **common + ) + await factories.make_agent_message( + db_session, agent_id="hreftest", message_ts="13.0010", phase="new_post", + content="HREF-ROOT-B", sender_name="HrefBot", posted_at=2.0, **common + ) + await factories.make_agent_message( + db_session, agent_id="hrefmate", message_ts="13.0011", thread_ts="13.0010", + phase="thread_reply", content="HREF-REPLY-B", sender_name="HrefMateBot", **common + ) + await db_session.commit() + + page = await client.get("/agent/hreftest/conversations", headers=_auth(pi.id)) + assert page.status_code == 200 + assert 'href="/agent/hreftest/thread/13.0001"' in page.text + assert 'href="/agent/hreftest/thread/13.0010"' in page.text + assert page.text.count('href="/agent/hreftest/thread/') == 2, ( + "each root's badge must link to its own thread, not repeat one href for both" + ) diff --git a/tests/integration/test_full_run_live.py b/tests/integration/test_full_run_live.py new file mode 100644 index 0000000..2bebb72 --- /dev/null +++ b/tests/integration/test_full_run_live.py @@ -0,0 +1,1165 @@ +"""T13 — the whole system, running: real LLM turns and real Slack mirroring together. + +No other test runs `real_llm` and `live_slack` at once. Everything in the cohort tier +drives the engine with `NullTransport` (so a mirror that silently no-ops looks identical +from inside our own database — Rule S2), and everything in the Slack tier drives the +mirror with hand-written message text (so a scheduler or a prompt that never produces a +conversation looks identical from inside Slack). This module runs `SimulationEngine.start()` +— the real main loop, the real pollers, the real cohort gate, real Opus/Sonnet turns and +the real workspace — and then asks the one question the DB-primary design exists to answer: + + is there a message in one store that is not in the other? + +Four disciplines are inherited verbatim from `test_cohort_scenarios.py`, and each of them +cost a rewrite there: + +1. **The labs are complementary.** Every pair is a plausible collaboration, so nothing + the agents fail to do can be explained away by scientific irrelevance. +2. **The roster is trimmed to the agents under test**, and the workspace is collapsed to + ONE channel. Phase 1 keyword-matches profiles against seven seeded channels and Phase 5 + posts into whichever subscribed channel the model names; left alone, three agents + scatter and never meet, and every outcome claim comes back inconclusive. +3. **Harness-authored messages are recorded and excluded** from every "the agents + conversed" measurement. Counting them makes the claim true by construction. +4. **A run that produced no conversation is INCONCLUSIVE, not passing.** Said so in the + assertion message, because at 3am the difference matters. + +Two more are specific to running both dependencies at once: + +5. **Channel discovery is not stubbed.** `AgentSlackClient.list_channels` used to read one + 200-item page of 500+ conversations, ordered by channel id, which is not monotonic in + creation time — so asking Slack "does this channel exist" was a coin flip and + `_ensure_seeded_channels` would try to *create* the probe channel it had just failed to + see. Every client's `list_channels` was therefore replaced here with a fully paginated + stand-in. It paginates for itself now, and the stub is gone: the whole-system test is + the one place the engine's real bootstrap path can be observed, and a patch over the + function under test would make that impossible. The paginator is pinned directly by + `test_slack_client_live.py::test_list_channels_returns_every_public_channel` and, at + the engine level, by `test_slack_lifecycle_live.py:: + test_ensure_seeded_channels_adopts_a_channel_beyond_the_first_page`. +6. **Every outbound post is guarded to the probe channel.** `_phase5_new_post` reads + `action_data.get("channel", "general")` and posts there without checking it against the + target post's channel, so one malformed JSON reply from a model would write into the + workspace's real `#general`. The guard raises instead, and the count is asserted to be + zero — a violation is a finding, not a flake. + +The near-concluded seeded thread is a **precondition, not the claim**. Measured over 16 +real turns in the cohort tier, Phase 5 chose "skip" or "new post" almost every time and +produced zero threaded replies; waiting for a specific pair to spontaneously reach a +`:memo:`→✅ handshake makes the ThreadDecision assertion untestable rather than merely +slow. The seeded history is written through the real `_post_message` (so it exists in both +stores, like everything else) and is then rebuilt by the real resume path — which is the +normal case, not an edge case: `_rebuild_agent_state` runs on every restart. +""" + +import asyncio +import os +import re +import signal +import time +import uuid +from dataclasses import dataclass, field + +import pytest +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +import src.agent.simulation as sim +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.slack_client import ThreadNotFound, markdown_to_mrkdwn +from src.config import get_settings as real_settings +from src.models import ( + COHORT_ACTION_TOPOLOGY_SNAPSHOT, + AgentChannel, + AgentMessage, + AgentRegistry, + Cohort, + CohortAuditEvent, + CohortMembership, + LlmCallLog, + SimulationRun, + ThreadDecision, +) +from src.visibility import VISIBILITY_PUBLIC + +pytestmark = [ + pytest.mark.integration, + pytest.mark.live_slack, + pytest.mark.real_llm, + pytest.mark.skipif( + not os.environ.get("ANTHROPIC_API_KEY"), + reason="no ANTHROPIC_API_KEY — a full run costs real money and is opt-in", + ), +] + +AGENTS = ("su", "cravatt", "wiseman") + +# Complementary by construction (discipline 1). Every pair needs something only the +# other two have, so no pair can fail to converse for reasons of relevance. +LABS = { + "su": ( + "SuProbeBot", + "genome-scale CRISPR screens mapping E3 ligases to their substrates; we need " + "degrader chemistry to act on the hits and quantitative imaging to watch " + "substrate loss", + ), + "cravatt": ( + "CravattProbeBot", + "covalent chemoproteomics finding ligandable cysteines on E3 ligases and " + "elaborating them into degraders; we need screen hits worth targeting and an " + "imaging readout for degradation kinetics", + ), + "wiseman": ( + "WisemanProbeBot", + "quantitative single-cell imaging of substrate degradation kinetics and the " + "proteostasis stress response; we need screen hits to watch and degrader " + "chemistry to perturb them with", + ), +} + +# Slack's chat.postMessage is ~1 msg/s per channel. Only the harness's own seeding posts +# fast enough to need this; a real turn takes seconds of LLM time between posts. +POST_GAP = 1.1 + +# Bounds. The turn count is the primary one, enforced by wrapping `_run_turn` (a signal +# the loop itself does not expose). `budget_cap` is a real ceiling the engine enforces, but +# it is deliberately set clear of the turn bound rather than used to stop the run, because +# a run stopped by budget exhaustion WEDGES instead of exiting: +# +# `_turn_eligible` already excludes over-budget agents, so `_select_agent` returns None +# (and the loop breaks) only once EVERY agent is over budget. With exactly one agent +# still under budget, and that agent being `_last_llm_caller`, the loop takes the +# `continue` branch at simulation.py:504-518 forever: no turn is taken, `turn_count` +# never advances, `_last_llm_caller` is never cleared on that path, and nothing is +# logged above DEBUG. Measured: su=7/7, cravatt=7/7, wiseman=4/7 and the process spun +# until the wall-clock deadline. Reported, not fixed. +TURNS = int(os.environ.get("FULL_RUN_TURNS", "20")) +BUDGET = int(os.environ.get("FULL_RUN_BUDGET", "40")) +RESTART_TURNS_A = int(os.environ.get("FULL_RUN_RESTART_TURNS_A", "4")) +RESTART_TURNS_B = int(os.environ.get("FULL_RUN_RESTART_TURNS_B", "4")) +# api_call_count is rebuilt from llm_call_logs on resume (_rebuild_agent_state step 4), so +# the second engine's budget must be *cumulative* or it starts already exhausted. +RESTART_BUDGET_A = int(os.environ.get("FULL_RUN_RESTART_BUDGET_A", "40")) +RESTART_BUDGET_B = int(os.environ.get("FULL_RUN_RESTART_BUDGET_B", "80")) +# Wall-clock ceiling per engine. Enforced with request_stop(), never with a cancelling +# asyncio timeout: cancellation mid-await is the SIGKILL failure mode this file is about. +# Also the backstop for the livelock above, which is why it is not generous. +DEADLINE_S = float(os.environ.get("FULL_RUN_DEADLINE_S", "900")) + +# Slack splits a chat.postMessage `text` longer than this into several messages and +# returns the LAST chunk's ts. Measured against this workspace today: a 16,441-char post +# became five messages of 4000/4000/4000/4000/441 characters, and the returned ts was the +# fifth. See test_a_message_over_slacks_4000_char_limit_stays_in_bijection. +SLACK_TEXT_CHUNK = 4000 + +_LINK_LABELLED = re.compile(r"<(?:mailto:)?[^|>\s]+\|([^>]*)>") +_LINK_BARE = re.compile(r"<((?:https?|mailto):[^>\s]+)>") +_EMOJI_SHORTCODE = re.compile(r":[a-z0-9_+'\-]+:") +_NON_WORD = re.compile(r"[^\w\s]") +_WS = re.compile(r"\s+") + +_SYSTEM_SUBTYPES = { + "message_deleted", "message_changed", "channel_join", "channel_leave", + "channel_purpose", "channel_topic", "channel_name", "channel_archive", + "channel_unarchive", "bot_add", "bot_remove", +} + + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + + +@dataclass +class RunCtx: + """Everything a run needs, plus what the harness itself authored.""" + + factory: object + run_id: uuid.UUID + channel: str + channel_id: str + clients: dict + seed_ts: set = field(default_factory=set) + off_channel_posts: list = field(default_factory=list) + + +@dataclass +class TurnRecord: + """What the loop did, sampled per turn from inside _run_turn.""" + + turns: int = 0 + errors: list = field(default_factory=list) + gates: list = field(default_factory=list) + gate_active: list = field(default_factory=list) + preflight: list = field(default_factory=list) + deadline_hit: bool = False + + def diagnosis(self) -> str: + return ( + f"turns={self.turns} deadline_hit={self.deadline_hit} " + f"errors={self.errors} gate_active={set(self.gate_active)} " + f"preflight={set(self.preflight)}" + ) + + +@pytest.fixture +async def full_run(engine, slack_clients, slack_probe_channel, tmp_path, monkeypatch): + """A live workspace collapsed to one `t-` channel, a 3-agent roster, one cohort. + + Deliberately not the rolled-back ``db_session``: the engine opens its own sessions + and commits, and that is the path under test. + """ + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + name, cid = slack_probe_channel + + # Discipline 2: one channel, no keyword scatter. Rebound on the module because the + # engine reads these globals directly from a dozen call sites during a real turn. + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [name]) + monkeypatch.setattr(sim, "_UNIVERSAL_CHANNELS", {name}) + monkeypatch.setattr(sim, "_CHANNEL_KEYWORDS", {}) + + patched = real_settings().model_copy(update={ + "cohort_isolation_enabled": True, + "cohort_default_policy": "isolated", + "max_consecutive_reactive_turns": 3, + "turn_delay_seconds": 0.0, + "phase5_skip_probability": 0.0, + }) + monkeypatch.setattr(sim, "get_settings", lambda: patched) + + # Working memory is process-external state on a shared volume + # (profiles/memory/{agent}/public.md), written by `_update_agent_memory` on every + # thread closure and read back into every later prompt. Left at the real path, run N+1 + # inherits run N's conclusions: measured, a second run whose memory already said + # "closed: no_proposal" produced Phase 5 skips on nine consecutive turns. That biases + # a run toward INCONCLUSIVE, so isolate it per test. Both bindings are patched — + # simulation.py imports the constant by value (`from src.agent.agent import + # PROFILES_DIR`), so patching only the source module would leave the profile-mtime + # watcher pointed at the repo. + monkeypatch.setattr("src.agent.agent.PROFILES_DIR", tmp_path / "profiles") + monkeypatch.setattr(sim, "PROFILES_DIR", tmp_path / "profiles") + + ctx = RunCtx(factory=factory, run_id=run_id, channel=name, channel_id=cid, + clients=dict(slack_clients)) + + # Discipline 6: never write outside the probe channel. `list_channels` is + # deliberately NOT patched — see discipline 5 in the module docstring. + for client in slack_clients.values(): + monkeypatch.setattr( + client, "post_message", _channel_guard(client, name, cid, ctx.off_channel_posts) + ) + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for aid in AGENTS: + db.add(AgentRegistry(agent_id=aid, bot_name=LABS[aid][0], + pi_name=f"PI {aid}", status="active")) + cohort = Cohort(name="t13-one-cohort") + db.add(cohort) + await db.flush() + for aid in AGENTS: + db.add(CohortMembership(cohort_id=cohort.id, agent_id=aid)) + await db.commit() + + try: + yield ctx + finally: + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute( + delete(ThreadDecision).where(ThreadDecision.simulation_run_id == run_id) + ) + await db.execute( + delete(LlmCallLog).where(LlmCallLog.simulation_run_id == run_id) + ) + await db.execute( + delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id) + ) + await db.execute( + delete(AgentChannel).where(AgentChannel.simulation_run_id == run_id) + ) + # profile_revisions rows written by the memory update cascade from here. + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(AGENTS))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +def _canonical_text(text: str) -> str: + """Reduce a message to the content both stores can be held to. + + Slack does not store what you posted. Measured against this workspace, four rewrites + happen inside `chat.postMessage` before the text is ever readable back: + + 'See https://doi.org/10.1038/x' -> 'See ' + '✅ and ⏸️' -> ':white_check_mark: and :double_vertical_bar:' + 'Contact a@b.edu' -> 'Contact ' + 'half-life <2 hours & rising' -> 'half-life <2 hours & rising' + + The fourth is HTML escaping of ``<``, ``>`` and ``&``, and it is why this test could + not pass before: dropping punctuation reduces the authored ``<2`` to ``2`` but Slack's + ``<2`` to ``lt 2``, so any message containing an inequality or an ampersand + diverged. Scientific prose is full of them — "<2 hours", ">6 hours", "CRBN & VHL" — + so the first live run failed on an agent writing "say <2 hours". Verified by posting + both spellings to this workspace: a raw ``<`` comes back as ``<``, and an already + escaped ``<`` comes back unchanged. Unescaping first collapses the two spellings + onto one, exactly as the emoji handling below collapses its two. + + Asserting byte equality against all that pins Slack's own text normalisation, not our + mirror (Rule L2), and it fails the moment an agent cites a paper or types a check + mark — which is exactly what the ✅ close protocol asks it to do. So: undo Slack's + escaping, unwrap its link markup, drop emoji in *both* spellings (shortcode and the + raw codepoint), drop punctuation, and compare the remaining words. A mirror that + posted different content, truncated it, or swapped two messages still fails; Slack's + rendering no longer does. + """ + # Before _NON_WORD: afterwards '<' has already become the word 'lt'. Order + # matters more than it looks — &lt; must not double-unescape into '<', so this + # is a single pass over the three entities Slack actually emits, not html.unescape. + text = text.replace("<", "<").replace(">", ">").replace("&", "&") + text = _LINK_LABELLED.sub(r"\1", text) + text = _LINK_BARE.sub(r"\1", text) + text = _EMOJI_SHORTCODE.sub(" ", text) + text = text.encode("ascii", "ignore").decode("ascii") # raw emoji, smart quotes + text = _NON_WORD.sub(" ", text) + return _WS.sub(" ", text).strip().lower() + + +def _is_fragment_of(chunk: str, whole: str) -> bool: + """Is `chunk` a piece of `whole`? Used to tell a split fragment from a lost message. + + Compared on canonical text, and with the first and last token dropped: Slack cuts at + a fixed character count, so both ends of a chunk are usually half a word. + """ + words = _canonical_text(chunk).split() + if len(words) < 5: + return False + return " ".join(words[1:-1]) in _canonical_text(whole) + + +def _channel_guard(client, allowed_name, allowed_id, sink): + """Refuse (loudly) to post anywhere but the probe channel.""" + real = client.post_message + + def _post(channel, text, thread_ts=None): + if channel not in (allowed_name, allowed_id): + sink.append((client.agent_id, channel, (text or "")[:120])) + raise RuntimeError( + f"[{client.agent_id}] refusing to post outside the test channel: " + f"{channel!r} (would have written into the real workspace)" + ) + return real(channel, text, thread_ts=thread_ts) + + return _post + + +def _make_agents(): + agents = [] + for aid in AGENTS: + bot, summary = LABS[aid] + a = Agent(agent_id=aid, bot_name=bot, pi_name=f"PI {aid}") + # The cached-profile seam: a real profile without touching disk or the DB. + a._public_profile = f"# {aid.capitalize()} Lab\n\n{summary}\n" + a._private_profile = "No private instructions yet." + agents.append(a) + return agents + + +def _make_engine(ctx, *, budget, bare=False): + """A real engine with real Slack clients and slack_enabled=True. + + ``bare`` skips nothing in the engine — it only means the caller will drive + ``_post_message`` directly instead of ``start()``, so the channel maps that + ``_ensure_seeded_channels`` would populate are set up here instead. + """ + eng = SimulationEngine( + agents=_make_agents(), + slack_clients=ctx.clients, + max_runtime_minutes=0, + budget_cap=budget, + session_factory=ctx.factory, + # `--reset-cursors` (a real production flag), and it is load-bearing here. + # `_rebuild_agent_state` step 5 advances every agent's last_seen_cursor to + # max(posted_at), so on a resumed run the harness's own seeded intros are + # already "seen": Phase 2 returns nothing, interesting_posts stays empty and + # Phase 5 has nothing to reply to. Measured without it — turn 1 concluded the + # seeded thread and turns 2-5 were all "Agent chose to skip", then the loop + # went idle. Resetting the cursors is what lets three agents actually discover + # each other, which is the precondition for a multi-turn run to exist at all. + reset_cursors=True, + simulation_run_id=ctx.run_id, + slack_enabled=True, + ) + if bare: + eng.message_log.set_persist_callback(eng._enqueue_persist) + eng._channel_id_map = {ctx.channel: ctx.channel_id} + eng._channel_visibility = {ctx.channel: VISIBILITY_PUBLIC} + for a in eng.agents.values(): + a.state.subscribed_channels = {ctx.channel} + return eng + + +def _bound_turns(eng, rec, limit): + """Bound the loop by turn count and record per-turn gate state. + + The engine exposes no turn counter, and `budget_cap` alone is a blunt bound (a turn + costs 1-3 calls). Wrapping the bound method also gives us the per-turn errors that + `start()` otherwise only writes to the log. + """ + real = eng._run_turn + + async def _wrapped(agent): + rec.turns += 1 + rec.gates.append({ + a: (None if x.allowed_sender_ids is None else set(x.allowed_sender_ids)) + for a, x in eng.agents.items() + }) + rec.gate_active.append(eng._cohort_gate_active) + rec.preflight.append(eng._cohort_preflight_error) + try: + return await real(agent) + except Exception as exc: + rec.errors.append(f"{agent.agent_id}: {type(exc).__name__}: {exc}") + raise + finally: + if rec.turns >= limit: + eng.request_stop() + + eng._run_turn = _wrapped + + +async def _deadline(eng, rec, seconds): + """Graceful wall-clock ceiling. Never cancels an in-flight await.""" + try: + await asyncio.sleep(seconds) + except asyncio.CancelledError: + return + rec.deadline_hit = True + eng.request_stop() + + +async def _drive(eng, rec, *, turns, deadline=DEADLINE_S): + """Run the real main loop, bounded, and flush on the way out as main.py does.""" + _bound_turns(eng, rec, turns) + watchdog = asyncio.create_task(_deadline(eng, rec, deadline)) + try: + await eng.start() + finally: + watchdog.cancel() + await asyncio.gather(watchdog, return_exceptions=True) + + +# --------------------------------------------------------------------------- +# Seeding — preconditions, excluded from every measurement +# --------------------------------------------------------------------------- + + +def _last_ts(eng) -> str: + return eng.message_log._entries[-1].ts + + +async def _seed(ctx, *, replies: int) -> str: + """Write the intros and one near-concluded thread through the real mirror. + + Returns the thread's root ts. Both stores see all of it: the seeds are posted with + `_post_message`, exactly like an agent's own message, so they cannot themselves + create the one-store-only condition the tests are looking for. + + `replies` alternates cravatt/su and the LAST reply is su's `:memo: Summary`, which + leaves cravatt owing a reply to a thread that has an explicit proposal to confirm. + The models still choose what to do — ✅ (proposal), their own revised `:memo:`, or + ⏸️ (no_proposal) — and `max_thread_messages` closes it as `timeout` if they choose + none of them. What is seeded is the *opportunity*, not the outcome. + + Strictly two-party: the root tags CravattProbeBot, so + `MessageLog.get_thread_allowed_agents` pins the thread to {su, cravatt} and Phase 4 + aborts for anyone else. Seeding a third voice into it would build a thread the engine + then refuses to continue. wiseman is the third agent for a reason — it has to reach + the others through Phase 2/5 like a real participant. + """ + seeder = _make_engine(ctx, budget=0, bare=True) + + for aid in AGENTS: + await seeder._post_message( + aid, ctx.channel, + f":wave: Introducing our lab: {LABS[aid][1]}. Keen to hear from " + "complementary groups.", + ) + time.sleep(POST_GAP) + await seeder._flush_persisted() + + await seeder._post_message( + "su", ctx.channel, + ":bulb: Concretely: our CRISPR screen has 40 E3-ligase/substrate pairs with no " + "chemical entry point. @CravattProbeBot — what would you need from us to turn " + "one of those pairs into a degrader with a measured kinetic readout?", + ) + time.sleep(POST_GAP) + await seeder._flush_persisted() + root_ts = _last_ts(seeder) + + bodies = [ + ("cravatt", "Interested. We have covalent fragment hits on two of those " + "ligases already. What is the substrate turnover in your hands?"), + ("su", "Substrate half-life is 90 minutes in the unperturbed line; the screen " + "read out at 72 hours, so we cannot resolve kinetics ourselves."), + ("cravatt", "Then the kinetic readout is the bottleneck, not the chemistry. We " + "can supply two elaborated covalent handles at 1 and 10 micromolar " + "plus the inactive alkyne control."), + ("su", "We have a degron-tagged reporter line for that substrate, so a " + "live readout is possible on our side with the right imaging."), + ("cravatt", "Then the open question is whether engagement produces degradation " + "on the timescale of turnover, or only a stress response."), + ("su", "Right — the pooled screen cannot separate those two, which is exactly " + "why the 72-hour readout was uninformative."), + ("cravatt", "Four arms would settle it: two doses, the alkyne control, and " + "vehicle, all on your reporter line."), + ("su", ":memo: Summary — Proposal: CRISPR-nominated E3/substrate pair to " + "measured degradation kinetics.\n" + "What each lab brings: Su lab — validated E3/substrate pair and the " + "degron-tagged reporter line. Cravatt lab — two elaborated covalent " + "handles plus an inactive alkyne control.\n" + "Scientific question: does covalent engagement of the nominated E3 " + "produce substrate degradation on the timescale of substrate turnover, " + "or only a proteostasis stress response?\n" + "First experiment: four arms (2 compound doses, alkyne control, vehicle) " + "on the reporter line, read out live over 8 hours. Two weeks from " + "compound delivery.\n" + "Why together: neither the pooled screen nor bulk chemoproteomics can " + "resolve degradation on the timescale of turnover.\n" + "Confidence: [Moderate]"), + ] + for aid, body in bodies[:replies]: + await seeder._post_message(aid, ctx.channel, body, thread_ts=root_ts) + time.sleep(POST_GAP) + await seeder._flush_persisted() + + ctx.seed_ts = {e.ts for e in seeder.message_log._entries} + return root_ts + + +# --------------------------------------------------------------------------- +# Measurement — the two stores, read independently +# --------------------------------------------------------------------------- + + +async def _db_snapshot(ctx) -> dict[str, AgentMessage]: + async with ctx.factory() as db: + rows = (await db.execute( + select(AgentMessage) + .where(AgentMessage.simulation_run_id == ctx.run_id) + .order_by(AgentMessage.posted_at) + )).scalars().all() + return {r.message_ts: r for r in rows} + + +def _slack_snapshot(ctx) -> dict[str, tuple[str, str | None]]: + """``{ts: (text, thread_ts|None)}`` for every real message in the probe channel. + + Read through a client that did not author most of them, and enumerated + independently of the engine's own reconcile (history + one conversations.replies per + threaded root), so the two sides of the comparison are not the same code path. + """ + client = ctx.clients["su"] + out: dict[str, tuple[str, str | None]] = {} + for msg in client.get_full_channel_history(ctx.channel_id): + ts = msg.get("ts") + if not ts or msg.get("subtype") in _SYSTEM_SUBTYPES: + continue + out[ts] = (msg.get("text") or "", None) + if not msg.get("reply_count"): + continue + try: + replies = client.get_all_thread_replies(ctx.channel_id, ts) + except ThreadNotFound: + continue + for r in replies: + rts = r.get("ts") + if not rts or rts == ts or r.get("subtype") in _SYSTEM_SUBTYPES: + continue + out[rts] = (r.get("text") or "", ts) + return out + + +def _both_stores(ctx, db_rows): + """(db_only, slack_only) — retried once, because Slack search-side propagation can + lag a post by a second and a false headline is worse than a slow test.""" + slack = _slack_snapshot(ctx) + db_only = set(db_rows) - set(slack) + slack_only = set(slack) - set(db_rows) + if db_only or slack_only: + time.sleep(4.0) + slack = _slack_snapshot(ctx) + db_only = set(db_rows) - set(slack) + slack_only = set(slack) - set(db_rows) + return slack, db_only, slack_only + + +def _split_fragments(slack, db_rows, slack_only) -> dict[str, str]: + """``{fragment_ts: parent_row_ts}`` for Slack-only messages the >4000-char split + explains. Anything left over is a genuine one-store-only message.""" + out: dict[str, str] = {} + oversized = [ + (r.message_ts, markdown_to_mrkdwn(r.content)) for r in db_rows.values() + if len(markdown_to_mrkdwn(r.content)) > SLACK_TEXT_CHUNK + ] + for ts in slack_only: + for parent_ts, whole in oversized: + if _is_fragment_of(slack[ts][0], whole): + out[ts] = parent_ts + break + return out + + +async def _llm_calls(ctx) -> int: + async with ctx.factory() as db: + return (await db.execute( + select(func.count(LlmCallLog.id)) + .where(LlmCallLog.simulation_run_id == ctx.run_id) + )).scalar_one() + + +async def _decisions(ctx): + async with ctx.factory() as db: + return (await db.execute( + select(ThreadDecision).where(ThreadDecision.simulation_run_id == ctx.run_id) + )).scalars().all() + + +def _agent_authored(ctx, db_rows) -> list[AgentMessage]: + """Rows written by an agent during a turn — the harness's seeds excluded.""" + return [ + r for ts, r in db_rows.items() + if ts not in ctx.seed_ts and r.is_bot and r.agent_id + ] + + +# =========================================================================== +# T13.1 — the whole system, one pass +# =========================================================================== + + +async def test_a_full_run_keeps_both_stores_in_bijection(full_run): + """Real turns, real mirror, real gate — and no message in one store only. + + The headline assertion is set equality between `agent_messages.message_ts` for this + run and every ts Slack holds in the channel. In pure Slack-on mode the canonical id + *is* the Slack ts, so a row with no Slack twin means the mirror silently no-oped, and + a Slack message with no row means a message the DB-primary design has already lost. + + Everything else here is either a precondition for that assertion to mean anything + (a conversation actually happened) or a property the same run can pay for once: + the gate held, a thread concluded, a ThreadDecision was written, provenance recorded. + """ + ctx = full_run + root_ts = await _seed(ctx, replies=8) + seed_rows = await _db_snapshot(ctx) + assert len(seed_rows) == len(ctx.seed_ts) == 12, ( + f"seeding did not land: {len(seed_rows)} rows for {len(ctx.seed_ts)} seeds" + ) + + eng = _make_engine(ctx, budget=BUDGET) + rec = TurnRecord() + await _drive(eng, rec, turns=TURNS) + await eng.stop() + + db_rows = await _db_snapshot(ctx) + slack, db_only, slack_only = _both_stores(ctx, db_rows) + authored = _agent_authored(ctx, db_rows) + calls = await _llm_calls(ctx) + decisions = await _decisions(ctx) + where = ( + f"{rec.diagnosis()} llm_calls={calls} db={len(db_rows)} slack={len(slack)} " + f"agent_authored={len(authored)} decisions={[(d.outcome, d.thread_id) for d in decisions]}" + ) + + # --- the control: did anything actually happen? ------------------------- + assert not rec.errors, f"a turn raised: {rec.errors}. {where}" + assert rec.turns > 0, f"INCONCLUSIVE: the loop took no turns at all. {where}" + assert authored, ( + "INCONCLUSIVE, NOT PASSING: no agent wrote anything in " + f"{rec.turns} turns, so every store-consistency claim below is trivially " + f"true and this run proves nothing. {where}" + ) + replies_in_seeded_thread = [ + r for r in authored if r.thread_ts == root_ts + ] + assert replies_in_seeded_thread, ( + "INCONCLUSIVE, NOT PASSING: no real model wrote into the open thread, so no " + f"conversation formed and nothing about threading was exercised. {where}" + ) + + # --- the headline ------------------------------------------------------- + assert not db_only, ( + "MESSAGES IN POSTGRES WITH NO SLACK TWIN — the mirror no-oped for " + f"{len(db_only)} message(s): " + f"{[(t, db_rows[t].agent_id, db_rows[t].content[:60]) for t in sorted(db_only)]}. {where}" + ) + # This allowance is now expected to be EMPTY, and the cross-check below asserts it. + # It used to absorb the one benign explanation for a Slack-only message: a post over + # SLACK_TEXT_CHUNK arrived as several Slack messages and only the last one's ts was + # recorded, so the earlier chunks were in Slack with no row. The client cuts at the + # boundary itself now and the engine records one row per message, so no row can be + # over the limit and `_split_fragments` therefore finds nothing to excuse. Kept + # rather than deleted because it is self-neutralising — `oversized` empty forces + # `fragments` empty — and it names, at the point of use, what a regression looks + # like. Anything that is NOT a fragment of a message we do have is a genuine loss. + fragments = _split_fragments(slack, db_rows, slack_only) + unexplained = slack_only - set(fragments) + assert not unexplained, ( + "MESSAGES IN SLACK WITH NO POSTGRES ROW — the primary store lost " + f"{len(unexplained)} message(s), and none of them is a >{SLACK_TEXT_CHUNK}-char " + f"split fragment of a message it does have: " + f"{[(t, slack[t][0][:80]) for t in sorted(unexplained)]}. {where}" + ) + assert set(db_rows) == set(slack) - set(fragments), ( + f"stores disagree beyond the characterised split. fragments={fragments}. {where}" + ) + + # --- and the mapping is usable, not merely present ---------------------- + oversized = [] + for ts, row in db_rows.items(): + assert row.slack_ts == ts, ( + f"row {ts} records slack_ts={row.slack_ts!r}; in Slack-on mode the " + f"canonical id IS the Slack ts. {where}" + ) + assert row.slack_channel_id == ctx.channel_id, ( + f"row {ts} points at channel {row.slack_channel_id!r}, not {ctx.channel_id}" + ) + assert row.channel_name == ctx.channel, ( + f"row {ts} claims channel #{row.channel_name}, outside the test channel" + ) + assert row.content.strip() and slack[ts][0].strip(), ( + f"row {ts}: an empty message reached one of the stores " + f"(db={row.content[:40]!r} slack={slack[ts][0][:40]!r})" + ) + full = markdown_to_mrkdwn(row.content) + if len(full) > SLACK_TEXT_CHUNK: + # Split by Slack: the row holds the whole message, this ts holds its tail. + oversized.append((ts, len(full))) + assert _is_fragment_of(slack[ts][0], full), ( + f"row {ts} is over the {SLACK_TEXT_CHUNK}-char limit and its Slack twin " + f"is not even a piece of it.\n db: {row.content[:200]!r}\n" + f" slack: {slack[ts][0][:200]!r}" + ) + else: + assert _canonical_text(full) == _canonical_text(slack[ts][0]), ( + f"row {ts} and its Slack twin do not carry the same content.\n" + f" db: {row.content[:200]!r}\n slack: {slack[ts][0][:200]!r}" + ) + # Threading agrees on both sides, translated through the mirror mapping. + assert slack[ts][1] == row.slack_thread_ts, ( + f"row {ts}: db slack_thread_ts={row.slack_thread_ts!r}, " + f"Slack says thread_ts={slack[ts][1]!r}" + ) + if row.thread_ts: + assert row.slack_thread_ts == row.thread_ts, ( + f"reply {ts} has a canonical parent {row.thread_ts} that differs from " + f"its Slack parent {row.slack_thread_ts} — no message here was minted " + "with Slack off, so they must agree" + ) + + assert ctx.off_channel_posts == [], ( + "an agent tried to post outside the test channel; _phase5_new_post defaults " + f"`channel` to 'general' when the model omits it: {ctx.off_channel_posts}" + ) + # Cross-check the allowance against its cause: a fragment may only exist because a + # message was over the limit, and an over-limit message must produce fragments. + assert bool(fragments) == bool(oversized), ( + f"the split allowance and its cause disagree: fragments={fragments} " + f"oversized={oversized}. {where}" + ) + + # --- the gate held throughout ------------------------------------------- + expected_gate = set(AGENTS) + assert rec.gates, f"no gate sample was taken. {where}" + for i, sample in enumerate(rec.gates): + assert sample == {a: expected_gate for a in AGENTS}, ( + f"the cohort gate changed at turn {i}: {sample}. {where}" + ) + assert set(rec.gate_active) == {True}, ( + f"the gate reported itself inactive during the run: {rec.gate_active}. {where}" + ) + assert set(rec.preflight) == {None}, ( + f"the cohort preflight forced isolation off mid-run: {rec.preflight}. {where}" + ) + assert eng._cohort_tags_stripped == {}, ( + "an in-cohort @mention was stripped — every agent here shares a cohort with " + f"every other, so the strip count must be zero: {eng._cohort_tags_stripped}" + ) + + async with ctx.factory() as db: + snaps = (await db.execute( + select(CohortAuditEvent).where( + CohortAuditEvent.action == COHORT_ACTION_TOPOLOGY_SNAPSHOT, + CohortAuditEvent.simulation_run_id == ctx.run_id, + ) + )).scalars().all() + assert snaps, f"the run recorded no topology snapshot, so it is unattributable. {where}" + assert snaps[0].topology["cohort_isolation_enabled"] is True + assert snaps[0].topology["agents"]["su"] == sorted(AGENTS) + + # --- a thread concluded, and the decision is durable -------------------- + assert decisions, ( + "no ThreadDecision was written: the thread neither reached a :memo:/✅ " + "handshake, nor a ⏸️ close, nor the max_thread_messages backstop. " + f"{where}" + ) + decided = [d for d in decisions if d.thread_id == root_ts] + assert decided, ( + f"a thread concluded but not the one under test: " + f"{[(d.thread_id, d.outcome) for d in decisions]}. {where}" + ) + d = decided[0] + assert d.outcome in ("proposal", "no_proposal", "timeout"), d.outcome + assert {d.agent_a, d.agent_b} <= set(AGENTS) + assert d.channel == ctx.channel + if d.outcome == "proposal": + assert d.summary_text and ":memo:" in d.summary_text, ( + f"a proposal was recorded with no summary: {d.summary_text!r}" + ) + # The conclusion must not have cost the mirror its consistency: re-check the + # closed thread's Slack twin explicitly, since _close_thread runs after the post. + assert eng._closed_thread_ids >= {root_ts} + + +# =========================================================================== +# T13.1b — the one-store-only condition, isolated and deterministic +# +# Found by the run above, then reduced to these two tests: no LLM calls, four Slack +# calls, and a definite answer. The >4000-char case was an xfail(strict=True) here while +# the split defect was open, which is why the run test above carries a split-fragment +# allowance; both are now closed and the pair is the cheapest end-to-end evidence for it. +# =========================================================================== + + +async def test_a_short_message_round_trips_one_to_one(full_run): + """Control for the test below: the mirror IS in bijection for ordinary messages. + + Without this leg, a failure below is equally explained by the mirror being broken for + everything, or by the probe channel being unreadable (Rule S2). + """ + ctx = full_run + eng = _make_engine(ctx, budget=0, bare=True) + await eng._post_message("su", ctx.channel, "one ordinary message, well under the limit") + time.sleep(POST_GAP) + await eng._flush_persisted() + + db_rows = await _db_snapshot(ctx) + slack, db_only, slack_only = _both_stores(ctx, db_rows) + assert len(db_rows) == 1, [r.content for r in db_rows.values()] + assert len(slack) == 1, slack + assert not db_only and not slack_only, (db_only, slack_only) + row = next(iter(db_rows.values())) + assert row.slack_ts == row.message_ts + assert _canonical_text(slack[row.message_ts][0]) == _canonical_text(row.content) + + +async def test_a_message_over_slacks_4000_char_limit_stays_in_bijection(full_run): + """One `_post_message` produces one row per Slack message it really made. + + Was `xfail(strict=True)`. Slack splits a `chat.postMessage` `text` over 4000 + characters into several messages itself and returns only the LAST chunk's ts; + `post_message` passed that single ts back, `_post_message` recorded it as the + canonical id, and every earlier chunk existed in Slack with no `agent_messages` row. + The consequences went past a missing row — `slack_ts` named the *tail*, so + `_slack_parent_ts` threaded replies onto a fragment, `posted_at = float(ts)` took the + tail's clock, and the next restart's `_rebuild_state_from_slack` saw the unrecorded + head chunks as brand-new inbound messages and ingested them. + + Phase 4 replies are generated with `max_tokens=1500`, roughly 6000 characters, so + this is reached by ordinary agent traffic: it is what the 20-turn run tripped over. + The client now cuts at the boundary itself and reports every message it created, and + the engine writes one row each — so the set equality below is exact, with no + "characterised split fragment" allowance on either side. + """ + ctx = full_run + eng = _make_engine(ctx, budget=0, bare=True) + body = "Opening sentence of a long reply. " + " ".join( + f"Point {i} on covalent degrader kinetics and single-cell imaging." for i in range(1, 121) + ) + assert len(body) > SLACK_TEXT_CHUNK, len(body) + + await eng._post_message("su", ctx.channel, body) + time.sleep(POST_GAP) + await eng._flush_persisted() + + db_rows = await _db_snapshot(ctx) + slack, db_only, slack_only = _both_stores(ctx, db_rows) + row = next(iter(db_rows.values())) + detail = ( + f"posted {len(body)} chars; Slack holds {len(slack)} message(s) of lengths " + f"{sorted(len(t) for t, _ in slack.values())}; the DB holds {len(db_rows)} row(s) " + f"of lengths {sorted(len(r.content) for r in db_rows.values())}; the first row " + f"recorded slack_ts={row.slack_ts} which is the " + f"{'LAST' if row.slack_ts == max(slack) else 'first' if row.slack_ts == min(slack) else 'nth'}" + f" of them; {len(slack_only)} chunk(s) have no row" + ) + assert not db_only, detail + assert set(db_rows) == set(slack), detail + + # The split really happened — otherwise the bijection above is the trivial one and + # this test would pass just as well against a client that refused to post at all. + assert len(slack) > 1, detail + # Every row is a message Slack accepted whole: nothing over the limit survives, so + # nothing was silently re-split on Slack's side behind our back. + assert all(len(markdown_to_mrkdwn(r.content)) <= SLACK_TEXT_CHUNK + for r in db_rows.values()), detail + # One logical post stays ONE top-level post. Without this, the continuations arrive + # as N fresh roots and every other agent's Phase 2 scan sees N posts for one. + roots = [r for r in db_rows.values() if r.thread_ts is None] + assert len(roots) == 1, ( + f"the split produced {len(roots)} top-level posts: " + f"{[(r.message_ts, r.content[:40]) for r in roots]}" + ) + assert row.slack_ts == min(slack), ( + "the recorded canonical id is not the FIRST Slack message — a reply threaded on " + f"it would hang off a fragment. {detail}" + ) + # And the whole post survived the split: no chunk lost, none duplicated. + rejoined = re.sub(r"\s+", "", "".join( + r.content for r in sorted(db_rows.values(), key=lambda r: float(r.message_ts)) + )) + assert rejoined == re.sub(r"\s+", "", body), ( + "the rows do not reassemble into the posted message" + ) + + +# =========================================================================== +# T13.2 — SIGTERM, restart, and the property the DB-primary design exists for +# =========================================================================== + + +# NOT xfailed. It was, on the reasoning that this test's phase B builds a fresh +# engine so the _rebuild_agent_state idempotency fixes could not have addressed +# it — reasoning made without credentials to check it. Run live for the first time +# on 2026-08-04 with all three probe bots, it PASSED, and the strict xfail turned +# that into a failure, which is the marker doing its job. The defect 8515f65 +# recorded is fixed; the pin is gone rather than relaxed. +async def test_sigterm_and_restart_lose_nothing_and_duplicate_nothing(full_run): + """Stop the engine with a real SIGTERM mid-turn, resume the same run, compare stores. + + Three separable claims, and the middle one is the one that has never been tested with + Slack on: + + 1. **The signal path works.** `main.py` installs `loop.add_signal_handler(SIGTERM, + request_stop)`; the same wiring is installed here and a real `SIGTERM` is delivered + to this process while a turn is in flight. The loop finishes the turn, flushes, and + returns — it is not cancelled. + 2. **`stop()`'s flush is load-bearing.** A message is posted after the loop has + exited: it reaches Slack synchronously but lives only in `_pending_persist`. The + negative control asserts it is NOT yet in Postgres — that is exactly what + `docker rm -f` (SIGKILL) destroys — and then that `stop()` recovers it. + 3. **Resume neither loses nor duplicates.** A second engine takes the same + `simulation_run_id`, rebuilds from the DB, reconciles against Slack, and runs more + turns. Every ts from before the restart must still be there, with unchanged + content, exactly once; no Slack message may be ingested twice under a second + canonical id; and the two stores must still be in bijection at the end. + """ + ctx = full_run + root_ts = await _seed(ctx, replies=3) + + # ---------------- phase A: run, then SIGTERM mid-turn ------------------ + eng1 = _make_engine(ctx, budget=RESTART_BUDGET_A) + rec1 = TurnRecord() + _bound_turns(eng1, rec1, 10 ** 6) # bounded by the signal, not by a counter + + loop = asyncio.get_running_loop() + fired: list[float] = [] + + def _shutdown(): # byte-for-byte the intent of main.py's handler + eng1.request_stop() + + loop.add_signal_handler(signal.SIGTERM, _shutdown) + + async def _sigterm_when_running(): + while rec1.turns < RESTART_TURNS_A: + await asyncio.sleep(0.2) + fired.append(time.time()) + os.kill(os.getpid(), signal.SIGTERM) + + killer = asyncio.create_task(_sigterm_when_running()) + watchdog = asyncio.create_task(_deadline(eng1, rec1, DEADLINE_S)) + try: + await eng1.start() + finally: + for t in (killer, watchdog): + t.cancel() + await asyncio.gather(killer, watchdog, return_exceptions=True) + loop.remove_signal_handler(signal.SIGTERM) + + assert fired, ( + f"the watchdog never reached turn {RESTART_TURNS_A}, so no SIGTERM was sent. " + f"{rec1.diagnosis()}" + ) + assert not rec1.errors, f"a turn raised before the signal: {rec1.errors}" + assert eng1._running is False and eng1._stop_event.is_set(), ( + "SIGTERM did not reach request_stop() — the loop exited for some other reason" + ) + buffered_at_exit = len(eng1._pending_persist) + + # Claim 2: what SIGKILL would have destroyed. + marker = f"post-signal, pre-flush {uuid.uuid4().hex[:8]}" + await eng1._post_message("su", ctx.channel, marker) + time.sleep(POST_GAP) + assert eng1._pending_persist, "the post did not buffer, so the control is vacuous" + pre_flush = await _db_snapshot(ctx) + assert marker not in [r.content for r in pre_flush.values()], ( + "the message was already durable, so this control cannot show what the " + "shutdown flush saves" + ) + slack_now = _slack_snapshot(ctx) + assert any(marker in t for t, _ in slack_now.values()), ( + "the message never reached Slack either, so nothing was at risk" + ) + await eng1.stop() + after_flush = await _db_snapshot(ctx) + assert marker in [r.content for r in after_flush.values()], ( + "stop() did not flush the buffered message — a graceful shutdown loses the " + "in-flight turn, which is the whole reason CLAUDE.md forbids `docker rm -f`" + ) + + db_a = await _db_snapshot(ctx) + decided_a = {d.thread_id for d in await _decisions(ctx)} + slack_a, db_only_a, slack_only_a = _both_stores(ctx, db_a) + frag_a = _split_fragments(slack_a, db_a, slack_only_a) + authored_a = _agent_authored(ctx, db_a) + where_a = ( + f"phaseA turns={rec1.turns} buffered_at_exit={buffered_at_exit} " + f"db={len(db_a)} slack={len(slack_a)} authored={len(authored_a)} " + f"split_fragments={len(frag_a)}" + ) + assert not db_only_a, f"DB-only before the restart: {sorted(db_only_a)}. {where_a}" + assert not (slack_only_a - set(frag_a)), ( + "Slack-only before the restart, and not explained by the >4000-char split: " + f"{sorted(slack_only_a - set(frag_a))}. {where_a}" + ) + assert authored_a, ( + "INCONCLUSIVE, NOT PASSING: nothing was written by an agent before the " + f"signal, so the restart has nothing at risk to preserve. {where_a}" + ) + + # ---------------- phase B: resume the same simulation_run_id ----------- + eng2 = _make_engine(ctx, budget=RESTART_BUDGET_B) + rec2 = TurnRecord() + rebuilt: dict[str, set] = {} + + # _backfill_foa_cache is the first setup step after all three rebuild passes + # (DB -> Slack reconcile -> agent state), so it is where "what did resume + # reconstruct" can be read before any new turn muddies it. + original_backfill = eng2._backfill_foa_cache + + async def _snapshot_after_rebuild(): + rebuilt["log"] = {e.ts for e in eng2.message_log._entries} + rebuilt["threads"] = { + aid: set(a.state.active_threads) for aid, a in eng2.agents.items() + } + rebuilt["calls"] = {aid: a.api_call_count for aid, a in eng2.agents.items()} + return await original_backfill() + + eng2._backfill_foa_cache = _snapshot_after_rebuild + await _drive(eng2, rec2, turns=RESTART_TURNS_B) + await eng2.stop() + + db_b = await _db_snapshot(ctx) + slack_b, db_only_b, slack_only_b = _both_stores(ctx, db_b) + authored_b = _agent_authored(ctx, db_b) + where = ( + f"{where_a} | phaseB turns={rec2.turns} db={len(db_b)} slack={len(slack_b)} " + f"authored={len(authored_b)} errors={rec2.errors} " + f"rebuilt_log={len(rebuilt.get('log', ()))}" + ) + + # Claim 3a: resume reconstructed exactly what was stored — no loss, no phantoms. + # The one permitted extra is a >4000-char split fragment: it is in Slack with no row, + # so `_rebuild_state_from_slack` legitimately treats it as a message the DB is missing + # and ingests it. That is the compounding cost of the split defect (each restart turns + # the unrecorded head chunks into first-class messages), not a rebuild bug. + assert rebuilt, f"resume never reached the rebuild snapshot point. {where}" + missing = set(db_a) - rebuilt["log"] + invented = rebuilt["log"] - set(db_a) + assert not missing, ( + "the resumed engine's message log is missing rows it rebuilt from: " + f"{sorted(missing)}. {where}" + ) + assert invented <= set(frag_a), ( + "resume invented messages that are not even split fragments of a stored one: " + f"{sorted(invented - set(frag_a))}. {where}" + ) + # Conversational state, not just message rows, must survive. Which thread is a + # measured outcome rather than an assumption: `_rebuild_agent_state` deliberately + # skips threads that already have a ThreadDecision, and the seeded thread often + # concludes during phase A — so requiring *that* thread back is asserting the + # opposite of correct behaviour. Measured once: phase A concluded the seeded thread + # and restored the two threads the agents had opened themselves. + restored = {t for s in rebuilt["threads"].values() for t in s} + assert restored, ( + "no open thread survived the restart on any agent, so nothing about conversational " + f"state was preserved. threads={rebuilt['threads']} decided={decided_a}. {where}" + ) + assert root_ts in restored or root_ts in decided_a, ( + f"the seeded thread {root_ts} neither came back as an open thread nor concluded — " + f"it was silently dropped. restored={restored} decided={decided_a}. {where}" + ) + assert not (restored & decided_a), ( + f"a concluded thread was reopened by the rebuild: {restored & decided_a}. {where}" + ) + assert sum(rebuilt["calls"].values()) > 0, ( + "api_call_count did not survive the restart, so the resumed run's budget is " + f"reset and a restart loop could spend without limit: {rebuilt['calls']}" + ) + + # Claim 3b: nothing lost. + lost = set(db_a) - set(db_b) + assert not lost, ( + f"the restart LOST {len(lost)} message(s) from Postgres: " + f"{[(t, db_a[t].content[:60]) for t in sorted(lost)]}. {where}" + ) + lost_slack = set(slack_a) - set(slack_b) + assert not lost_slack, f"messages vanished from Slack: {sorted(lost_slack)}. {where}" + + # Claim 3c: nothing duplicated. The failure mode is specific — the Slack reconcile + # re-ingesting a message it already has under a *different* canonical id — so the + # test is on slack_ts uniqueness, not on the (constraint-enforced) message_ts. + slack_ts_seen: dict[str, list[str]] = {} + for ts, row in db_b.items(): + if row.slack_ts: + slack_ts_seen.setdefault(row.slack_ts, []).append(ts) + dupes = {s: v for s, v in slack_ts_seen.items() if len(v) > 1} + assert not dupes, ( + f"one Slack message is represented by several rows: {dupes}. {where}" + ) + frag_b = _split_fragments(slack_b, db_b, slack_only_b) + assert not db_only_b, f"DB-only after the restart: {sorted(db_only_b)}. {where}" + assert not (slack_only_b - set(frag_b)), ( + "Slack-only after the restart, and not explained by the >4000-char split: " + f"{sorted(slack_only_b - set(frag_b))}. {where}" + ) + assert set(db_b) == set(slack_b) - set(frag_b), ( + f"stores disagree beyond the characterised split. fragments={frag_b}. {where}" + ) + + # Claim 3d: and the surviving rows were not rewritten by the resume. The flush + # upserts with ON CONFLICT DO UPDATE, so a rebuild that reconstructed an entry + # slightly differently would silently clobber the original. + for ts, before in db_a.items(): + after = db_b[ts] + assert after.content == before.content, ( + f"row {ts} was rewritten across the restart:\n before {before.content[:120]!r}" + f"\n after {after.content[:120]!r}" + ) + assert (after.agent_id, after.thread_ts, after.is_bot, after.channel_name) == ( + before.agent_id, before.thread_ts, before.is_bot, before.channel_name + ), f"row {ts} metadata changed across the restart" + + assert ctx.off_channel_posts == [], ctx.off_channel_posts + assert set(rec2.preflight) <= {None}, ( + f"the gate turned itself off after the restart: {rec2.preflight}. {where}" + ) + for sample in rec2.gates: + assert sample == {a: set(AGENTS) for a in AGENTS}, ( + f"the gate did not survive the restart intact: {sample}. {where}" + ) diff --git a/tests/integration/test_grantbot_live.py b/tests/integration/test_grantbot_live.py new file mode 100644 index 0000000..2fcc6f1 --- /dev/null +++ b/tests/integration/test_grantbot_live.py @@ -0,0 +1,1581 @@ +"""GrantBot's funding flow — live grants.gov, the real Anthropic API, real Postgres. (T10) + +`tests/unit/test_funding_rules.py` and `tests/unit/test_grantbot_lead_time.py` cover the +pure functions. `tests/live_api/test_grants_live.py` covers the grants.gov client. What +neither can see is the flow: a real opportunity, as grants.gov returns it *today*, +travelling through the lead-time filter, the selection LLM, the draft LLM, the +`grantbot_posted_foas` claim and into a stored funding message. Before this file +`src/models/grantbot_posted.py` — the primitive that stops GrantBot posting the same FOA +twice — was referenced by no test at all. + +**Slack is never touched.** Another agent owns the test workspace. Every test either +forces the Slack-off path or installs a recording double in place of `slack_sdk.WebClient` +(`_RecordingWebClient`), and the Slack-off tests install `_ExplodingWebClient`, which +fails the test if GrantBot so much as constructs a client. + +**What is stubbed, and why.** The ceiling for this task is 25 Anthropic calls. Two tests +spend real tokens because only a real model can answer their question (`real_llm`): +whether a live FOA survives selection and comes back as a usable post, and whether the +`funding_rules` regexes — written against imagined phrasing — actually classify prose a +model writes. The dedup, lead-time and Slack-transport tests replace GrantBot's two LLM +stages with `_StageRecorder`, which is not a compromise but the sharper instrument: it +records *which opportunities reached each stage*, which is precisely the claim those +tests make, and it makes the assertion depend on the filter rather than on model +judgement. Everything else in those tests — grants.gov, the close dates, the database, +the claim — is real. + +**Facts this file is built on** (established by the T3 agent against live grants.gov, not +re-derived here): + +- `search2` never returns a `description`. `search_opportunities` therefore always yields + `description=""` and `grantbot.py:306` feeds that empty string to the drafting LLM. + Known, reported, deliberately unfixed — pinned by + `test_the_draft_prompt_is_built_from_an_empty_description` so it cannot silently change + in either direction. +- `fetchOpportunity`'s backend is currently returning an outage envelope (HTTP 200, + `errorcode: 0`, `data.message` = backend unavailable), so `fetch_opportunity_detail` + returns None. Tests that depend on detail data say "provider is down" explicitly rather + than passing quietly. +- Live close dates are `MM/DD/YYYY`. An unparseable close date returns None, and + `_has_sufficient_lead_time` treats None as "rolling" and PASSES. That asymmetry means a + date-format change disables lead-time filtering entirely, silently — characterized by + `test_an_unparseable_close_date_turns_the_lead_time_filter_off`. + +Run: + + docker compose exec -T -e LIVE_API_TESTS=1 -e ANTHROPIC_API_KEY=sk-ant-... \\ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_b3 \\ + app python -m pytest tests/integration/test_grantbot_live.py -q -m live_api +""" + +import asyncio +import json +import os +import re +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +import pytest_asyncio +from sqlalchemy import select + +from src.agent import grantbot +from src.agent.funding_rules import ( + is_acknowledgment_only_funding_reply, + is_announcement_only_funding_reply, + summarize_funding_thread, +) +from src.agent.message_log import LogEntry, MessageLog +from src.agent.slack_client import SLACK_MAX_TEXT_CHARS, split_for_slack +from src.models import AgentMessage, GrantbotPostedFoa, SimulationRun +from src.services import grants + +# The whole module is the live tier: every test reads today's grants.gov catalogue. +pytestmark = [pytest.mark.integration, pytest.mark.live_api] + +needs_llm = pytest.mark.skipif( + not os.environ.get("ANTHROPIC_API_KEY"), + reason="no ANTHROPIC_API_KEY — real-API tests are opt-in and cost money", +) + +# `list_posted_opportunities` pages at 250 internally and never sees the rate limiter; +# charge the budget for the pages it is about to request. +_PAGE_SIZE = 250 + +# The six channels grantbot's drafting prompt offers the model. A seventh would be +# posted to a channel that does not exist in the workspace. +ALLOWED_CHANNELS = { + "drug-repurposing", "structural-biology", "aging-and-longevity", + "single-cell-omics", "chemical-biology", "funding-opportunities", +} + +# Mechanism + subject-matter filters used to choose live FOAs the selection prompt is +# meant to keep. Rule L2: these select *a* qualifying opportunity from today's catalogue, +# never a named one, so nothing here goes stale when an FOA closes. +_MECHANISM_RE = re.compile(r"\((R01|R21|R35|U01|U19|P01|R33|DP1|DP2)\b", re.IGNORECASE) +_BIOMEDICAL_RE = re.compile( + r"\b(cancer|immun\w*|neuro\w*|virus|viral|infect\w*|protein\w*|structural|genom\w*|" + r"proteom\w*|drug|therapeut\w*|molecul\w*|cell\w*|microbi\w*|aging|biolog\w*|" + r"chemi\w*|discovery|metabol\w*)\b", + re.IGNORECASE, +) +# The drafting prompt's own EXCLUDE list — feeding GrantBot one of these and then +# complaining that it was not selected would be testing the model's obedience to a rule +# we asked it to follow. +_EXCLUDED_RE = re.compile( + r"\b(training|fellowship|T32|F31|F32|K\d\d|career|conference|supplement|scholar|" + r"education|diversity|small business|SBIR|STTR)\b", + re.IGNORECASE, +) + +# Words that appear in every FOA title and therefore prove no grounding. +_BOILERPLATE = { + "clinical", "trial", "optional", "required", "allowed", "research", "program", + "grants", "grant", "award", "awards", "initiative", "opportunity", "limited", + "competition", "national", "institute", "institutes", "notice", "funding", +} + + +# --------------------------------------------------------------------------- helpers + + +def content_words(text: str, min_len: int = 6) -> set[str]: + """Distinctive lowercase words in `text` — boilerplate and short words removed.""" + words = re.findall(rf"[A-Za-z][A-Za-z\-]{{{min_len - 1},}}", text) + return {w.lower() for w in words} - _BOILERPLATE + + +def days_out(opp: dict, now: datetime) -> int | None: + """Days from `now` to the opportunity's close date, or None if unparseable/empty.""" + close = grantbot._parse_close_date(opp.get("close_date", "")) + return None if close is None else (close - now).days + + +def synthetic_opportunity(number: str, now: datetime) -> dict: + """An FOA shaped exactly like `search_opportunities` returns one. + + Every other test in this file feeds GrantBot a *live* opportunity, because their + claims are about the live feed: its date formats, its empty `description`, whether a + real FOA survives selection. The two splitting tests below make no claim about + grants.gov at all — their subject is how many Slack messages an 11,000-character body + becomes — so spending catalogue budget and a `fetchOpportunity` round trip on them + would buy nothing and would make a splitting test fail when the feed was down. + """ + return { + "number": number, + "id": "999999", + "title": "Synthetic Mechanisms of Long Bodies (R01 Clinical Trial Not Allowed)", + "agency": "HHS-NIH11", + "close_date": ( + now + timedelta(days=grantbot.MIN_LEAD_DAYS + 60) + ).strftime("%m/%d/%Y"), + "description": "", + "synopsis": "", + } + + +def expected_header(opp: dict) -> str: + """The header `_run_grantbot_with_session` prepends to every funding post. + + Duplicated from grantbot.py on purpose: it is the part of the message that is NOT + model output, and pinning it here is how a change to the FOA number, close date or + grants.gov link that agents cite becomes a test failure instead of a silent edit. + """ + return ( + ":moneybag: *Funding Opportunity*\n" + f"*{opp.get('title', '')}*\n" + f"{opp.get('number', 'unknown')} | Closes: {opp.get('close_date', 'Not specified')}\n" + f"https://www.grants.gov/search-results-detail/{opp.get('id', '')}\n\n" + ) + + +class _StageRecorder: + """Stands in for GrantBot's two LLM stages and records what reached each. + + Only installed by tests whose claim is about the *filters*, not about model output: + what a test of the lead-time cut or the dedup claim needs to observe is which + opportunities arrived at the selection and drafting stages, and a real model's + include/exclude judgement would only add noise to that. + """ + + def __init__(self, channel: str = "funding-opportunities", body: str | None = None): + self.channel = channel + # `body` overrides the stub draft text. The splitting tests need a body of a + # chosen length, and the length of a *real* model's draft is not something a test + # about splitting can control — `_draft_post` caps max_tokens at 500. + self.body = body + self.offered_to_select: list[str] = [] + self.drafted: list[str] = [] + self.select_calls = 0 + + async def select(self, opportunities: dict, max_select: int = 30) -> list[str]: + self.select_calls += 1 + self.offered_to_select.extend(opportunities) + return list(opportunities)[:max_select] + + async def draft(self, opportunity: dict) -> dict: + number = opportunity.get("number", "") + self.drafted.append(number) + return { + "channel": self.channel, + "post_text": self.body if self.body is not None else ( + f"Stubbed draft body for {number}. Scope, mechanism, eligibility." + ), + } + + def install(self, monkeypatch): + monkeypatch.setattr(grantbot, "_select_opportunities", self.select) + monkeypatch.setattr(grantbot, "_draft_post", self.draft) + return self + + +class _ExplodingWebClient: + """Any construction is a test failure: T10 must never reach a Slack workspace.""" + + def __init__(self, *args, **kwargs): + raise AssertionError( + "GrantBot reached a real Slack transport during a Slack-OFF test — it would " + "have posted into the shared copi-test workspace, which another agent owns. " + "`slack_globally_enabled` was patched to False, so reaching here means the " + "gate in _run_grantbot_with_session no longer consults it." + ) + + +class _RecordingWebClient: + """A Slack transport double. Records posts; never opens a socket. + + `next_fail_post` makes `chat_postMessage` raise, which is how the claim-release path + (a post that failed must not leave the FOA marked as posted) gets exercised. It is + read *per call* off the class rather than captured at construction because GrantBot + now posts through `src.services.slack_web`, whose `_client` seam the `slack_on` + fixture backs with a single shared double for a whole test: a flag captured in + `__init__` would be frozen at whatever it was when that one instance was built, and + the failure half of the transport test — which flips the flag between two runs — + would silently exercise the success path instead. + """ + + instances: list["_RecordingWebClient"] = [] + + def __init__(self, token: str = "", **kwargs): + assert not token.startswith("xoxb-") or "fake" in token, ( + f"the Slack double was handed {token[:12]!r} — a test must never pass a " + "real bot token anywhere near a transport, even a fake one" + ) + self.token = token + self.posts: list[dict] = [] + self.joined: list[str] = [] + self.listed: list[dict] = [] + _RecordingWebClient.instances.append(self) + + next_fail_post = False + + def conversations_list(self, **kwargs): + self.listed.append(dict(kwargs)) + return { + "channels": [{"name": n, "id": f"C{n[:8].upper()}"} for n in ALLOWED_CHANNELS], + "response_metadata": {"next_cursor": ""}, + } + + def conversations_join(self, channel: str): + self.joined.append(channel) + return {"ok": True} + + def chat_postMessage(self, channel: str, text: str): + if _RecordingWebClient.next_fail_post: + raise RuntimeError("simulated Slack outage") + self.posts.append({"channel": channel, "text": text}) + return {"ok": True, "ts": f"1700000000.{len(self.posts):06d}"} + + +class _SettingsWithFakeToken: + """Real settings, with the two Slack bot tokens replaced by an obvious fake. + + Belt and braces: the transport is already a double, but this guarantees that even a + regression that bypassed the double could not authenticate as a real bot. + """ + + def __init__(self, real, token: str = "xoxb-fake-t10-token"): + self._real, self._token = real, token + + def __getattr__(self, name): + if name in ("slack_bot_token_grantbot", "slack_bot_token_su"): + return self._token + return getattr(self._real, name) + + +# --------------------------------------------------------------------------- fixtures + + +@pytest.fixture(autouse=True) +def _isolate_foa_cache(monkeypatch, tmp_path): + """`cache_foa` writes into the repo's data/ directory. Redirect it at a tmp dir.""" + monkeypatch.setattr("src.agent.foa_cache.CACHE_DIR", tmp_path / "foa_cache") + + +@pytest.fixture(scope="session") +def now_utc() -> datetime: + """One clock for the whole module, so a boundary FOA cannot flip mid-session.""" + return datetime.now(UTC) + + +@pytest.fixture(scope="session") +def live_catalogue(api_budget) -> list[dict]: + """Today's posted NIH/NSF opportunities, fetched once and shared. + + Rule L3: an empty catalogue is grants.gov being down or its `data.oppHits` path + moving, not a property of GrantBot — say so rather than letting every test below + fail on an unrelated symptom. + """ + for _ in range(4): # the client pages internally at 250/request + api_budget.wait("grants") + opportunities = asyncio.run(grants.list_posted_opportunities()) + if not opportunities: + pytest.fail( + "PROVIDER: grants.gov returned zero posted " + f"{grants.BIOMEDICAL_AGENCIES} opportunities. Every test in this module " + "reads that catalogue, so nothing below can be concluded. This is not a " + "GrantBot failure." + ) + return opportunities + + +@pytest.fixture(scope="session") +def biomedical_candidates(live_catalogue, now_utc) -> list[dict]: + """Live NIH opportunities the selection prompt is designed to keep. + + Comfortably past the lead-time cut (MIN_LEAD_DAYS + 30) so the selection test cannot + fail for a lead-time reason, and sorted by FOA number so a rerun within the same day + exercises the same opportunities. + """ + out = [ + o for o in live_catalogue + if o.get("agency") == "HHS-NIH11" + and o.get("id") + and (d := days_out(o, now_utc)) is not None + and d >= grantbot.MIN_LEAD_DAYS + 30 + and _MECHANISM_RE.search(o.get("title", "")) + and _BIOMEDICAL_RE.search(o.get("title", "")) + and not _EXCLUDED_RE.search(o.get("title", "")) + ] + out.sort(key=lambda o: o["number"]) + return out + + +@pytest_asyncio.fixture +async def sim_run(db_session) -> SimulationRun: + """A simulation run for GrantBot's DB post to land in, asserted to be the latest. + + `_post_funding_to_db` calls `get_latest_run_id`, which orders by `started_at`. If a + stale run in this database were newer, the funding post would be filed against it and + every assertion below would look for the message in the wrong run. + """ + from src.services.pi_inbox import get_latest_run_id + + run = SimulationRun( + id=uuid.uuid4(), + started_at=datetime.now(UTC) + timedelta(seconds=5), + status="running", + config={"source": "tests/integration/test_grantbot_live.py"}, + ) + db_session.add(run) + await db_session.flush() + latest = await get_latest_run_id(db_session) + assert latest == run.id, ( + f"get_latest_run_id returned {latest}, not the run this test just created " + f"({run.id}) — a newer simulation_runs row exists in this database and GrantBot " + "would post into it" + ) + return run + + +@pytest.fixture +def slack_off(monkeypatch): + """Force the DB-post path and make any real Slack client construction fatal. + + Both seams are stopped because GrantBot's transport moved: it posts through + `src.services.slack_web`, whose only `WebClient` construction is `_client`, and + `slack_web` binds `WebClient` at *import* time — so patching `slack_sdk.WebClient` + alone would no longer stop anything. That patch is kept anyway: it is what catches a + regression that goes back to constructing a client inside grantbot.py itself, which is + precisely the bypass `tests/unit/test_slack_boundary.py` exists to forbid. + """ + async def _disabled(db): + return False + + monkeypatch.setattr("src.services.slack_tokens.slack_globally_enabled", _disabled) + monkeypatch.setattr("slack_sdk.WebClient", _ExplodingWebClient) + monkeypatch.setattr("src.services.slack_web._client", _ExplodingWebClient) + + +@pytest.fixture +def slack_on(monkeypatch): + """Take the Slack branch, but through `_RecordingWebClient` with a fake token. + + `slack_web._client` is backed by *one* shared double for the whole test, so + `client.posts` and `client.joined` stay a single ledger. Production builds a fresh + `WebClient` per boundary call; nothing asserted here depends on that, and a shared + ledger is what lets a test count the messages one FOA produced. + """ + async def _enabled(db): + return True + + _RecordingWebClient.instances = [] + _RecordingWebClient.next_fail_post = False + + def _shared_double(token: str = "", **kwargs) -> _RecordingWebClient: + if not _RecordingWebClient.instances: + _RecordingWebClient(token) + return _RecordingWebClient.instances[0] + + monkeypatch.setattr("src.services.slack_tokens.slack_globally_enabled", _enabled) + monkeypatch.setattr("slack_sdk.WebClient", _RecordingWebClient) + monkeypatch.setattr("src.services.slack_web._client", _shared_double) + real_settings = grantbot.get_settings() + monkeypatch.setattr(grantbot, "get_settings", lambda: _SettingsWithFakeToken(real_settings)) + return _RecordingWebClient + + +@pytest.fixture +def llm_calls(monkeypatch): + """Record every real Anthropic call GrantBot makes, without changing its behaviour. + + The recording wrapper is how the drafting prompt gets inspected: the prompt is built + inside `_draft_post` and is otherwise unobservable, and it is where the empty + `description` ends up. + """ + from src.services import llm as llm_service + + real = llm_service.generate_agent_response + calls: list[dict] = [] + + async def recording(system_prompt, messages, **kwargs): + response = await real(system_prompt=system_prompt, messages=messages, **kwargs) + calls.append({ + "phase": (kwargs.get("log_meta") or {}).get("phase"), + "system": system_prompt, + "user": messages[-1]["content"] if messages else "", + "response": response, + }) + return response + + monkeypatch.setattr(llm_service, "generate_agent_response", recording) + return calls + + +@pytest.fixture +def fixed_catalogue(monkeypatch): + """Serve GrantBot a chosen slice of the live catalogue. + + The opportunities are real and were fetched from grants.gov moments earlier; only + *how many* of them GrantBot sees is controlled, because the unbounded pipeline drafts + one LLM call per selected opportunity (up to 30) and this task has a 25-call ceiling. + """ + def _install(opportunities: list[dict], budget=None): + async def _listed(agencies=None): + return [dict(o) for o in opportunities] + + monkeypatch.setattr(grantbot, "list_posted_opportunities", _listed) + if budget is not None: + for _ in opportunities: # the pipeline fetches detail per selected opp + budget.wait("grants") + + return _install + + +async def _messages_for_run(db_session, run_id) -> list[AgentMessage]: + rows = await db_session.execute( + select(AgentMessage) + .where(AgentMessage.simulation_run_id == run_id) + .order_by(AgentMessage.posted_at) + ) + return list(rows.scalars().all()) + + +async def _claimed_numbers(db_session) -> set[str]: + rows = await db_session.execute(select(GrantbotPostedFoa.foa_number)) + return set(rows.scalars().all()) + + +# --------------------------------------------------------------------------- the flow + + +@pytest.mark.real_llm +@needs_llm +async def test_a_live_opportunity_flows_through_to_a_drafted_funding_message( + db_session, sim_run, biomedical_candidates, llm_calls, slack_off, + fixed_catalogue, api_budget, +): + """A real FOA from today's grants.gov reaches agent_messages as a usable post. + + Two live opportunities in, a real selection call, a real drafting call each, and a + row in the database at the other end. Everything between is production code. + + Control (against the failure this test exists to catch): the drafted body must share + a distinctive word with the live FOA title. A pipeline that lost its input and had + the model write a plausible generic funding post would satisfy every structural + assertion here and fail that one. + + Second control: the rerun at the end costs zero LLM calls, which proves the + already-posted pre-filter runs *before* the model rather than after it. + """ + candidates = biomedical_candidates[:2] + assert len(candidates) == 2, ( + f"only {len(candidates)} live NIH opportunities matched " + f"{_MECHANISM_RE.pattern} + biomedical wording with >= " + f"{grantbot.MIN_LEAD_DAYS + 30} days of runway. grants.gov's catalogue is " + "unusually thin today or the agency/mechanism filters no longer match its " + "titles — this is about the catalogue, not about GrantBot" + ) + fixed_catalogue(candidates, budget=api_budget) + + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", + dry_run=False, max_posts=5, max_per_channel=5, + ) + + select_calls = [c for c in llm_calls if c["phase"] == "select"] + draft_calls = [c for c in llm_calls if c["phase"] == "draft"] + assert len(select_calls) == 1, ( + f"expected exactly one selection call, saw {len(select_calls)} " + f"(phases seen: {[c['phase'] for c in llm_calls]})" + ) + assert posted, ( + "the pipeline posted nothing. The selection LLM was offered " + f"{[o['number'] + ': ' + o['title'][:60] for o in candidates]} and returned " + f"{select_calls[0]['response'][:200]!r}; {len(draft_calls)} draft(s) followed. " + "If selection returned an empty array the model rejected live NIH R-series " + "biomedical FOAs, which is a prompt/model change, not a plumbing failure" + ) + assert len(draft_calls) >= len(posted), ( + f"{len(posted)} opportunities were posted but only {len(draft_calls)} draft calls " + "were made — a post went out with no model-written body" + ) + + by_number = {o["number"]: o for o in candidates} + messages = await _messages_for_run(db_session, sim_run.id) + assert len(messages) == len(posted), ( + f"the pipeline reported {len(posted)} post(s) {[p['number'] for p in posted]} but " + f"{len(messages)} agent_messages row(s) exist for this run — the DB write and the " + "return value disagree, so callers counting one are wrong about the other" + ) + + for record, message in zip(posted, messages, strict=True): + opportunity = by_number[record["number"]] + assert record["channel"] in ALLOWED_CHANNELS, ( + f"the drafting model chose channel {record['channel']!r}, which is not one of " + f"the six offered in its prompt ({sorted(ALLOWED_CHANNELS)}) — GrantBot would " + "post into a channel that does not exist" + ) + assert message.agent_id == "grantbot" and message.is_bot, ( + f"funding post stored as agent_id={message.agent_id!r} is_bot={message.is_bot} " + "— agents filter the log on both" + ) + assert message.phase == "new_post" and message.visibility == "public", ( + f"funding post stored with phase={message.phase!r} " + f"visibility={message.visibility!r}; funding threads are open to all and the " + "Phase 2 scan only sees top-level public posts" + ) + assert message.channel_name == record["channel"] + assert message.channel_id == f"local:{record['channel']}" + assert message.sender_name == "GrantBot" + assert message.message_ts and float(message.posted_at) > 0 + + header = expected_header(opportunity) + assert message.content.startswith(header), ( + "the funding post's header is not the one grantbot.py builds. Agents and " + "`_FOA_NUMBER_RE` in funding_rules.py read the number out of this header, and " + "the grants.gov link is what a PI clicks.\nexpected prefix:\n" + f"{header!r}\ngot:\n{message.content[:len(header) + 80]!r}" + ) + body = message.content[len(header):].strip() + assert len(body) > 120, ( + f"the model's post body for {record['number']} is {len(body)} chars " + f"({body!r}) — the summary a PI is meant to triage on is essentially empty" + ) + assert "**" not in body, ( + "the drafted body uses **double asterisks**, which Slack mrkdwn renders " + f"literally; the prompt forbids them explicitly. Body: {body[:300]!r}" + ) + assert not re.search(r"@\w+[Bb]ot\b", body), ( + "the drafted body tags a lab bot. The prompt forbids it ('lab agents will " + f"decide relevance themselves') and a tag skews who replies. Body: {body[:300]!r}" + ) + + shared = content_words(opportunity["title"]) & content_words(body) + assert shared, ( + f"the post drafted for {record['number']} shares no distinctive word with the " + f"live FOA title.\n title: {opportunity['title']!r}\n body: {body[:300]!r}\n" + "Either the opportunity never reached the prompt (the pipeline lost its input) " + "or the model wrote a generic funding post — both produce a post that " + "misrepresents the FOA to every PI who reads it" + ) + + claimed = await _claimed_numbers(db_session) + assert {p["number"] for p in posted} <= claimed, ( + f"posted {[p['number'] for p in posted]} but grantbot_posted_foas holds " + f"{sorted(claimed)} — nothing recorded the post, so the next run reposts it" + ) + row = (await db_session.execute( + select(GrantbotPostedFoa).where(GrantbotPostedFoa.foa_number == posted[0]["number"]) + )).scalar_one() + assert row.channel == posted[0]["channel"] and row.title == posted[0]["title"], ( + f"the claim row records channel={row.channel!r} title={row.title!r}, which does " + "not match what was posted" + ) + + # Rerun over exactly what was posted: the cheap pre-filter must empty the set before + # a single token is spent. + calls_before = len(llm_calls) + fixed_catalogue([by_number[p["number"]] for p in posted]) + again = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", + dry_run=False, max_posts=5, max_per_channel=5, + ) + assert again == [], f"the same opportunities posted a second time: {again}" + assert len(llm_calls) == calls_before, ( + f"the rerun spent {len(llm_calls) - calls_before} LLM call(s) on opportunities " + "already in grantbot_posted_foas — _load_posted_numbers is no longer filtering " + "before the model, which multiplies the cost of every daily run" + ) + assert len(await _messages_for_run(db_session, sim_run.id)) == len(messages), ( + "the rerun added an agent_messages row for an FOA that was already posted" + ) + + +# --------------------------------------------------------------------------- dedup + + +async def test_claim_foa_is_the_dedup_primitive(db_session): + """`models/grantbot_posted.py`, which no test referenced before this one. + + Absence and control interleaved: the second claim on the same number must fail, a + claim on a *different* number must succeed, and after `_release_foa` the first number + must be claimable again. A `_claim_foa` that always returned False would satisfy the + absence assertion on its own; it cannot satisfy the other two. + """ + number = f"TEST-T10-{uuid.uuid4().hex[:8].upper()}" + other = f"TEST-T10-{uuid.uuid4().hex[:8].upper()}" + + assert await grantbot._claim_foa(db_session, number, "funding-opportunities", "First"), ( + "the first claim on an unseen FOA number failed — INSERT ... ON CONFLICT DO " + "NOTHING reported rowcount 0 for a row that cannot have conflicted" + ) + assert not await grantbot._claim_foa(db_session, number, "chemical-biology", "Second"), ( + "the same FOA number was claimed twice. The foa_number primary key plus ON " + "CONFLICT DO NOTHING is the only thing stopping two GrantBot instances posting " + "the same opportunity, and it is not holding" + ) + assert await grantbot._claim_foa(db_session, other, "funding-opportunities", "Other"), ( + "CONTROL FAILED: a different, unseen FOA number was also refused — the claim is " + "rejecting everything, so the refusal above proves nothing about deduplication" + ) + + rows = (await db_session.execute( + select(GrantbotPostedFoa).where(GrantbotPostedFoa.foa_number == number) + )).scalars().all() + assert len(rows) == 1, f"{len(rows)} rows for one FOA number — the PK is not unique" + assert rows[0].channel == "funding-opportunities" and rows[0].title == "First", ( + "the losing claim overwrote the winner's channel/title; ON CONFLICT DO NOTHING " + "must not update" + ) + assert rows[0].posted_at is not None, "posted_at server_default did not fire" + + await grantbot._release_foa(db_session, number) + assert number not in await _claimed_numbers(db_session) + assert await grantbot._claim_foa(db_session, number, "aging-and-longevity", "Retry"), ( + "after _release_foa the number could not be re-claimed — a failed Slack post " + "would permanently retire the FOA instead of letting the next run retry it" + ) + + +async def test_the_claim_not_the_prefilter_is_what_stops_a_repost( + db_session, sim_run, biomedical_candidates, slack_off, fixed_catalogue, + monkeypatch, api_budget, +): + """Dedup holds even when the cheap pre-filter is defeated — and a new FOA still posts. + + Three runs over live opportunities with the LLM stages recorded rather than called: + + 1. FOA A posts. + 2. FOA A again, with `_load_posted_numbers` forced to return an empty set so the + pre-filter cannot hide the claim. The drafting stage must run (proving the + pre-filter really was bypassed) and nothing must be posted. + 3. CONTROL — FOA B, never seen, must post. Without it a `_claim_foa` that refused + everything, or a pipeline that had stopped posting entirely, would pass step 2. + """ + assert len(biomedical_candidates) >= 2, ( + f"need two live NIH opportunities, found {len(biomedical_candidates)}" + ) + first, second = biomedical_candidates[0], biomedical_candidates[1] + + # --- 1. first post + recorder = _StageRecorder().install(monkeypatch) + fixed_catalogue([first], budget=api_budget) + run_one = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert [p["number"] for p in run_one] == [first["number"]], ( + f"expected the live FOA {first['number']} to post, got {run_one}" + ) + assert await _claimed_numbers(db_session) == {first["number"]} + assert len(await _messages_for_run(db_session, sim_run.id)) == 1 + + # --- 2. same FOA, pre-filter defeated + async def _no_prefilter(session): + return set() + + monkeypatch.setattr(grantbot, "_load_posted_numbers", _no_prefilter) + recorder_two = _StageRecorder().install(monkeypatch) + fixed_catalogue([first], budget=api_budget) + run_two = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert recorder_two.drafted == [first["number"]], ( + "the drafting stage did not see the already-posted FOA, so the pre-filter was " + f"still in play and this run never reached the claim (drafted: " + f"{recorder_two.drafted}). The assertion below would prove nothing" + ) + assert run_two == [], ( + f"the FOA already in grantbot_posted_foas was posted again: {run_two}. With the " + "pre-filter bypassed, `_claim_foa` is the last line of defence and it did not hold" + ) + messages = await _messages_for_run(db_session, sim_run.id) + assert len(messages) == 1, ( + f"{len(messages)} funding messages exist for one FOA — the duplicate reached " + "agent_messages even though the claim was refused" + ) + claim_rows = (await db_session.execute( + select(GrantbotPostedFoa).where(GrantbotPostedFoa.foa_number == first["number"]) + )).scalars().all() + assert len(claim_rows) == 1 + + # --- 3. control: an unseen FOA still posts (pre-filter still bypassed) + recorder_three = _StageRecorder().install(monkeypatch) + fixed_catalogue([second], budget=api_budget) + run_three = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert [p["number"] for p in run_three] == [second["number"]], ( + f"CONTROL FAILED: the unseen live FOA {second['number']} did not post " + f"({run_three}). A dedup that blocks everything would have passed step 2 — until " + "this passes, step 2 means nothing" + ) + assert await _claimed_numbers(db_session) == {first["number"], second["number"]} + assert len(await _messages_for_run(db_session, sim_run.id)) == 2 + assert recorder.select_calls == recorder_three.select_calls == 1 + + +# --------------------------------------------------------------------------- lead time + + +async def test_lead_time_filtering_against_live_close_dates( + db_session, sim_run, live_catalogue, now_utc, slack_off, fixed_catalogue, + monkeypatch, api_budget, +): + """Both halves of the lead-time cut, against close dates grants.gov is serving today. + + The unit tests use hand-written dates. This one partitions the live catalogue and + pushes one FOA from each side through the real pipeline, so it fails if grants.gov + changes its date format, if MIN_LEAD_DAYS stops being applied, or if the filter is + applied to the wrong side. + + Rule L3: if grants.gov happens to have nothing closing inside the window today, the + reject half is unverifiable and this SKIPS with that reason rather than passing. The + two guards below exist because a *skip* is how this test would otherwise hide the two + changes it most needs to catch: the partition is drawn relative to MIN_LEAD_DAYS and + from parsed dates, so lowering the constant to zero or breaking the parser empties the + imminent side and turns a failure into a silent skip. Both were survivors in the + mutation run until these guards were added. + """ + assert grantbot.MIN_LEAD_DAYS >= 7, ( + f"MIN_LEAD_DAYS is {grantbot.MIN_LEAD_DAYS}. Below about a week the filter no " + "longer does the job it was added for — a lab cannot prepare a credible response " + "— and the imminent side of the partition below collapses, so this test would " + "SKIP rather than fail. If the constant was lowered deliberately, lower this " + "guard with it and say why" + ) + parseable = [o for o in live_catalogue if days_out(o, now_utc) is not None] + assert len(parseable) >= len(live_catalogue) * 0.5, ( + f"only {len(parseable)} of {len(live_catalogue)} live close_dates parse with " + "_parse_close_date. grants.gov changed its date format or the parser broke; every " + "FOA is now treated as rolling and the lead-time filter is off. Without this " + "assertion the empty partition below would SKIP and hide it" + ) + imminent = sorted( + (o for o in live_catalogue + if (d := days_out(o, now_utc)) is not None and 0 <= d <= grantbot.MIN_LEAD_DAYS - 3), + key=lambda o: (days_out(o, now_utc), o["number"]), + ) + roomy = [o for o in live_catalogue + if (d := days_out(o, now_utc)) is not None and d >= grantbot.MIN_LEAD_DAYS + 3] + if not imminent: + pytest.skip( + "CATALOGUE, not a failure: no posted grants.gov opportunity closes within " + f"{grantbot.MIN_LEAD_DAYS - 3} days today, so the reject half of the " + "lead-time filter cannot be exercised against a live date" + ) + assert roomy, ( + f"no posted opportunity closes more than {grantbot.MIN_LEAD_DAYS + 3} days out — " + "with no accept half, a filter that rejected everything would pass" + ) + short, long = imminent[0], sorted(roomy, key=lambda o: o["number"])[0] + + # The pure function, on live date strings rather than invented ones. + assert not grantbot._has_sufficient_lead_time( + short["close_date"], now_utc, grantbot.MIN_LEAD_DAYS + ), ( + f"{short['number']} closes {short['close_date']} " + f"({days_out(short, now_utc)} days out) and passed a " + f"{grantbot.MIN_LEAD_DAYS}-day lead-time filter" + ) + assert grantbot._has_sufficient_lead_time( + long["close_date"], now_utc, grantbot.MIN_LEAD_DAYS + ), ( + f"CONTROL FAILED: {long['number']} closes {long['close_date']} " + f"({days_out(long, now_utc)} days out) and was still rejected — the filter is " + "dropping everything, so the rejection above says nothing about lead time" + ) + + # The pipeline: the recorder shows exactly which FOA reached the model. + recorder = _StageRecorder().install(monkeypatch) + fixed_catalogue([short, long], budget=api_budget) + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + + assert short["number"] not in recorder.offered_to_select, ( + f"{short['number']} (closes {short['close_date']}, " + f"{days_out(short, now_utc)} days out) reached the selection stage. Step 2b of " + "_run_grantbot_with_session is meant to drop it — labs cannot prepare a credible " + "response in that time, and the money is spent scoring an FOA that cannot be used" + ) + assert long["number"] in recorder.offered_to_select, ( + f"CONTROL FAILED: {long['number']} (closes {long['close_date']}) did not reach " + "the selection stage either. The filter dropped both, so the exclusion above is " + "not evidence of lead-time filtering" + ) + assert [p["number"] for p in posted] == [long["number"]], ( + f"expected only {long['number']} to post, got {[p['number'] for p in posted]}" + ) + assert await _claimed_numbers(db_session) == {long["number"]}, ( + f"grantbot_posted_foas holds {sorted(await _claimed_numbers(db_session))} — an " + "FOA that was filtered out must not be claimed" + ) + + +async def test_an_unparseable_close_date_turns_the_lead_time_filter_off( + db_session, sim_run, live_catalogue, now_utc, slack_off, fixed_catalogue, + monkeypatch, api_budget, +): + """CHARACTERIZATION of a known asymmetry — this test asserts current behaviour, not + desired behaviour, and must not be "fixed" into passing differently. + + `_parse_close_date` returns None for anything outside %m/%d/%Y, %Y-%m-%d and + %Y/%m/%d, and `_has_sufficient_lead_time` reads None as "rolling submission, keep it". + Live dates are %m/%d/%Y. So the day grants.gov switches to an ISO timestamp or a + written-out month — a change no contract test can see, because those fixtures are + hand-written — every FOA becomes rolling, the lead-time filter stops filtering, and + nothing anywhere reports it. + + Same live opportunity, same real deadline, three renderings. The MM/DD/YYYY control + proves the filter works on the real feed; the other two show it switched off. + + The two guards repeat those in the test above for the same mutation-run reason: this + test selects its victim through `_parse_close_date` and `MIN_LEAD_DAYS`, so breaking + either would empty the selection and skip instead of failing. + """ + assert grantbot.MIN_LEAD_DAYS >= 7, ( + f"MIN_LEAD_DAYS is {grantbot.MIN_LEAD_DAYS} — too low to select an imminent FOA " + "with, so this test would SKIP rather than report that the filter was weakened" + ) + parseable = [o for o in live_catalogue if days_out(o, now_utc) is not None] + assert len(parseable) >= len(live_catalogue) * 0.5, ( + f"only {len(parseable)} of {len(live_catalogue)} live close_dates parse — the " + "format change this test *predicts* has happened; the lead-time filter is already " + "disabled in production and this test must not skip past it" + ) + imminent = sorted( + (o for o in live_catalogue + if (d := days_out(o, now_utc)) is not None and 0 <= d <= grantbot.MIN_LEAD_DAYS - 3), + key=lambda o: (days_out(o, now_utc), o["number"]), + ) + if not imminent: + pytest.skip( + "CATALOGUE, not a failure: nothing closes inside the lead-time window today, " + "so there is no imminent FOA to smuggle past the filter" + ) + victim = imminent[0] + real_close = grantbot._parse_close_date(victim["close_date"]) + assert real_close is not None, ( + f"{victim['close_date']!r} is no longer parseable — grants.gov has ALREADY " + "changed its date format and the lead-time filter is already disabled in " + "production. That is the failure this test predicts" + ) + + # Control: as grants.gov actually serves it, the filter rejects. + assert not grantbot._has_sufficient_lead_time( + victim["close_date"], now_utc, grantbot.MIN_LEAD_DAYS + ), f"CONTROL FAILED: {victim['number']} closing {victim['close_date']} was not rejected" + + plausible_reformats = { + "ISO 8601 with time": real_close.strftime("%Y-%m-%dT%H:%M:%SZ"), + "written-out month": real_close.strftime("%d %b %Y"), + "US long form": real_close.strftime("%B %d, %Y"), + } + for label, rendered in plausible_reformats.items(): + assert grantbot._parse_close_date(rendered) is None, ( + f"{label} ({rendered!r}) is parseable after all — update this test's list of " + "formats grants.gov could plausibly move to" + ) + assert grantbot._has_sufficient_lead_time( + rendered, now_utc, grantbot.MIN_LEAD_DAYS + ), ( + f"{label} no longer passes the lead-time filter. If _has_sufficient_lead_time " + "has been changed to reject unparseable dates, rolling/standing FOAs (which " + "legitimately have no deadline) are now being dropped — check that before " + "editing this test" + ) + + # And at the pipeline level: an FOA closing in `days` days walks straight through. + reformatted = dict(victim, close_date=real_close.strftime("%Y-%m-%dT%H:%M:%SZ")) + recorder = _StageRecorder().install(monkeypatch) + fixed_catalogue([reformatted], budget=api_budget) + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert recorder.offered_to_select == [victim["number"]], ( + "the reformatted date did NOT reach selection — behaviour has changed and the " + "asymmetry this test characterizes may be fixed. Re-read _has_sufficient_lead_time" + ) + assert [p["number"] for p in posted] == [victim["number"]], ( + "an FOA closing in " + f"{days_out(victim, now_utc)} days was not posted despite passing the filter" + ) + assert days_out(victim, now_utc) < grantbot.MIN_LEAD_DAYS, ( + "the opportunity chosen for this test is not actually imminent" + ) + + +# --------------------------------------------------------------------------- Slack leg + + +async def test_the_slack_leg_posts_through_a_double_and_releases_a_failed_claim( + db_session, sim_run, biomedical_candidates, slack_on, fixed_catalogue, + monkeypatch, api_budget, +): + """The Slack branch, exercised without a Slack workspace. + + Both outcomes, because they are the two halves of one invariant — a claim exists iff + the post landed: + + - the post succeeds: `chat_postMessage` is called with the full text and the claim stays; + - the post raises: `_release_foa` removes the claim so the next run can retry. + + Also pins a real asymmetry: on the Slack branch GrantBot writes NOTHING to + agent_messages. CLAUDE.md states the DB, not Slack, is the durable store, and every + other writer in the system persists first. + """ + assert biomedical_candidates, "no live NIH opportunity available" + opportunity = biomedical_candidates[0] + + recorder = _StageRecorder(channel="chemical-biology").install(monkeypatch) + fixed_catalogue([opportunity], budget=api_budget) + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + + assert len(slack_on.instances) == 1, ( + f"{len(slack_on.instances)} Slack clients were constructed for one run" + ) + client = slack_on.instances[0] + assert [p["number"] for p in posted] == [opportunity["number"]] + assert len(client.posts) == 1, ( + f"the Slack branch made {len(client.posts)} chat_postMessage call(s) for one " + f"opportunity: {client.posts}" + ) + sent = client.posts[0] + assert sent["channel"] == "#chemical-biology", ( + f"posted to {sent['channel']!r}; the drafted channel must be sent with a leading " + "'#', which is how the WebClient resolves a name rather than an id" + ) + assert sent["text"].startswith(expected_header(opportunity)), ( + f"the Slack text does not start with the funding header:\n{sent['text'][:250]!r}" + ) + assert opportunity["number"] in sent["text"] and recorder.drafted == [opportunity["number"]] + assert client.joined, ( + "the bot never called conversations_join — GrantBot cannot post to a public " + "channel it has not joined, so the first run in a fresh workspace would fail" + ) + assert client.listed and all( + call.get("exclude_archived") is True and call.get("types") == "public_channel" + for call in client.listed + ), ( + f"the auto-join listing was requested as {client.listed}. Both arguments are " + "deliberate and neither is the boundary's default: the map feeds " + "conversations_join, an archived channel cannot be joined, and a private channel " + "cannot be joined by a bot that was never invited. `list_channel_ids` defaults " + "exclude_archived to False because its other callers ask 'does this name exist', " + "where an archived channel still owns its name — so this call site has to pass it" + ) + assert await _claimed_numbers(db_session) == {opportunity["number"]}, ( + "a successful Slack post left no grantbot_posted_foas row — the next run reposts it" + ) + assert await _messages_for_run(db_session, sim_run.id) == [], ( + "GrantBot wrote a funding post to agent_messages on the Slack branch. That is not " + "current behaviour (see _run_grantbot_with_session step 6, which returns straight " + "after chat_postMessage); if it has changed, this pin should change with it" + ) + + # --- the failure half: a raising transport must release the claim + _RecordingWebClient.next_fail_post = True + other = biomedical_candidates[1] + _StageRecorder(channel="chemical-biology").install(monkeypatch) + fixed_catalogue([other], budget=api_budget) + failed = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert failed == [], f"a failed Slack post was reported as posted: {failed}" + assert other["number"] not in await _claimed_numbers(db_session), ( + f"{other['number']} is still claimed after chat_postMessage raised — the FOA is " + "permanently retired: never posted, never retried. _release_foa did not run" + ) + assert await _claimed_numbers(db_session) == {opportunity["number"]}, ( + "CONTROL FAILED: the successful claim was released too, so the release above is " + "not evidence that failures specifically are rolled back" + ) + + +# ------------------------------------------------------------------ splitting (T13) + +# 37 characters per unit; 320 units is ~11.8k characters, which is three Slack messages +# at SLACK_MAX_TEXT_CHARS and cuts cleanly on word boundaries. A body that produced only +# two chunks would let an off-by-one in the splitter pass. +_LONG_BODY = "Funding opportunity detail sentence. " * 320 + + +def _stub_detail_fetch(monkeypatch) -> None: + """Make step 4's `fetch_opportunity_detail` a no-op, so no test here calls out. + + The synthetic opportunity carries an `id`, which is what `_run_grantbot_with_session` + checks before fetching detail. Leaving the `id` off would also skip the fetch, but it + would change the grants.gov URL in the header and make the header assertions below + quietly weaker than they look. + """ + async def _no_detail(opportunity_id: str): + return None + + monkeypatch.setattr(grantbot, "fetch_opportunity_detail", _no_detail) + + +async def test_a_long_funding_post_is_split_into_messages_slack_will_accept( + db_session, sim_run, now_utc, slack_on, fixed_catalogue, monkeypatch, +): + """>4000 characters must leave GrantBot as N messages, not as one Slack will chunk. + + T13 measured Slack silently splitting an over-long body and returning only the last + ts. GrantBot posted through a raw `WebClient` with no splitting, so it could not + learn that: it logged one post and had one ts's worth of nothing, while the workspace + held three messages — and the simulation's channel poller, which is what mirrors + GrantBot's Slack posts into `agent_messages` (`simulation.py`, the `is_bot` branch of + the channel poll), ingested three rows. Routing the post through + `slack_web.post_message` splits it here, so GrantBot's count, Slack's count and the + mirror's count are the same number by construction. + + `split_for_slack` is used to compute the expected count rather than a hard-coded 3: + the number of chunks is a property of the splitter, and hard-coding it would make + this test fail if the splitter's boundary heuristics changed for a good reason. The + `>= 3` guard below is what stops that making the assertion vacuous. + """ + _stub_detail_fetch(monkeypatch) + opportunity = synthetic_opportunity("TEST-SPLIT-SLACK", now_utc) + _StageRecorder(channel="chemical-biology", body=_LONG_BODY).install(monkeypatch) + fixed_catalogue([opportunity]) + + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert [p["number"] for p in posted] == [opportunity["number"]], ( + f"the long FOA did not post at all: {posted}" + ) + + full_post = expected_header(opportunity) + _LONG_BODY + chunks = split_for_slack(full_post, SLACK_MAX_TEXT_CHARS) + assert len(chunks) >= 3 and len(full_post) > 2 * SLACK_MAX_TEXT_CHARS, ( + f"the test body is only {len(full_post)} characters and splits into " + f"{len(chunks)} chunk(s) — too short to distinguish splitting from not splitting" + ) + + client = _RecordingWebClient.instances[0] + sent = [p["text"] for p in client.posts] + assert len(sent) == len(chunks), ( + f"GrantBot made {len(sent)} chat_postMessage call(s) for a {len(full_post)}-" + f"character post that Slack accepts as {len(chunks)} messages. Slack does not " + "reject the oversized call — it splits it and returns only the last ts, so the " + "divergence is silent" + ) + for text in sent: + assert len(text) <= SLACK_MAX_TEXT_CHARS, ( + f"a {len(text)}-character chunk was sent; Slack's limit is " + f"{SLACK_MAX_TEXT_CHARS} and it splits anything longer itself" + ) + assert all(p["channel"] == "#chemical-biology" for p in client.posts), ( + f"chunks went to more than one channel: {[p['channel'] for p in client.posts]}" + ) + assert sent[0].startswith(expected_header(opportunity)), ( + f"the first chunk is not the head of the post:\n{sent[0][:250]!r}" + ) + # Joined on whitespace, not on "": `split_for_slack` rstrips/lstrips at each cut (the + # whitespace a chunk boundary lands on is the boundary), so concatenating the chunks + # directly fuses the last word of one to the first word of the next. Comparing word + # sequences is the guarantee the splitter actually documents — no non-whitespace + # character lost or duplicated. + assert " ".join(sent).split() == full_post.split(), ( + "splitting lost, duplicated or reordered content — the words that reached Slack " + "are not the words GrantBot drafted" + ) + assert await _claimed_numbers(db_session) == {opportunity["number"]}, ( + "a successful split post left no grantbot_posted_foas row — the next run reposts it" + ) + assert await _messages_for_run(db_session, sim_run.id) == [], ( + "GrantBot wrote funding rows to agent_messages on the Slack branch. It must not: " + "the simulation's channel poller already ingests GrantBot's Slack posts, keyed by " + "the Slack ts, and a second row minted with a local canonical id would not dedup " + "against it — every funding post would reach the agents twice, and threads rooted " + "on the local copy would never reach Slack (see the `is_bot` branch of the channel " + "poll in simulation.py). If this ever should change, the poller's dedup has to " + "change with it" + ) + + +async def test_a_long_funding_post_becomes_one_db_row_per_slack_message( + db_session, sim_run, now_utc, slack_off, fixed_catalogue, monkeypatch, +): + """Slack-off: N rows of at most 4000 characters, not one row of 11,800. + + This is the other half of the same invariant. `_post_funding_to_db` stored the whole + body in a single `agent_messages` row, so the same content was one message in the + database and three in Slack — `split_for_slack`'s docstring calls a chunk-per-row the + thing "that puts agent_messages in bijection with Slack", and a single oversized row + breaks it. The expected count is the *same* `split_for_slack` count the Slack test + above asserts against, which is what ties the two branches to one number. + """ + _stub_detail_fetch(monkeypatch) + opportunity = synthetic_opportunity("TEST-SPLIT-DB", now_utc) + _StageRecorder(body=_LONG_BODY).install(monkeypatch) + fixed_catalogue([opportunity]) + + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert [p["number"] for p in posted] == [opportunity["number"]], ( + f"the long FOA did not post at all: {posted}" + ) + + full_post = expected_header(opportunity) + _LONG_BODY + chunks = split_for_slack(full_post, SLACK_MAX_TEXT_CHARS) + assert len(chunks) >= 3, f"test body splits into only {len(chunks)} chunk(s)" + + rows = await _messages_for_run(db_session, sim_run.id) + assert len(rows) == len(chunks), ( + f"{len(rows)} agent_messages row(s) for a {len(full_post)}-character funding post " + f"that Slack would render as {len(chunks)} messages" + ) + for row in rows: + assert len(row.content) <= SLACK_MAX_TEXT_CHARS, ( + f"a row holds {len(row.content)} characters — over Slack's " + f"{SLACK_MAX_TEXT_CHARS} limit, so mirroring it would split it silently" + ) + assert [r.content for r in rows] == chunks, ( + "the stored rows are not the split chunks in order" + ) + assert len({r.message_ts for r in rows}) == len(rows), ( + "two chunks share a canonical message_ts — mint_local_ts was called once and " + "reused, and uq_agent_messages_run_ts will drop one of the rows" + ) + assert all(r.sender_name == "GrantBot" and r.is_bot for r in rows), ( + "a chunk was filed under a different author than the post it came from" + ) + assert all(r.channel_name == "funding-opportunities" for r in rows), ( + f"chunks landed in {sorted({r.channel_name for r in rows})}" + ) + assert all(r.phase == "new_post" for r in rows), ( + "a chunk was stored as a thread_reply; every chunk is a top-level post, which is " + "what Phase 2 scans" + ) + assert await _claimed_numbers(db_session) == {opportunity["number"]}, ( + "the FOA was not claimed, so a later run would post it again" + ) + + +async def test_a_chunk_that_fails_leaves_no_half_written_funding_post( + db_session, sim_run, now_utc, slack_off, fixed_catalogue, monkeypatch, +): + """A failure partway through a split post must leave zero rows and zero claims. + + Splitting opened this window. When one FOA was one row, a failed write left nothing + behind. One row per chunk means a failure can land after chunk 1 and before chunk 4 — + and the recovery path is `_release_foa`, which **commits**. So without a rollback the + fragment is committed, the claim is released, and the next run posts the whole FOA on + top of it: agents then read two chunks of one post and four of another, which is the + divergence splitting exists to remove. + + The `written == 1` assertion is the control. A fault injector that raised on the + *first* chunk would leave nothing to roll back, and the two assertions below would + pass against an implementation that never rolls anything back. + """ + _stub_detail_fetch(monkeypatch) + opportunity = synthetic_opportunity("TEST-SPLIT-FAIL", now_utc) + _StageRecorder(body=_LONG_BODY).install(monkeypatch) + fixed_catalogue([opportunity]) + # Read the id out before the run. The recovery path rolls the session back, and + # rollback expires every loaded ORM object regardless of expire_on_commit — touching + # `sim_run.id` afterwards would trigger a lazy reload and raise MissingGreenlet. This + # is a property of holding an ORM handle across the rollback, which only a test does: + # GrantBot's loop carries plain dicts from here on. + run_id = sim_run.id + + real_write = grantbot._post_funding_to_db + written: list[str] = [] + + async def _fail_after_the_first_chunk(session, channel_name, text): + if written: + raise RuntimeError("simulated DB failure partway through a split post") + written.append(text) + await real_write(session, channel_name, text) + + monkeypatch.setattr(grantbot, "_post_funding_to_db", _fail_after_the_first_chunk) + + posted = await grantbot._run_grantbot_with_session( + db_session, channel="funding-opportunities", dry_run=False, + max_posts=5, max_per_channel=5, + ) + assert posted == [], f"a funding post that failed halfway was reported as posted: {posted}" + assert len(written) == 1, ( + f"CONTROL FAILED: {len(written)} chunk(s) were written before the injected " + "failure. The point of this test is a *partial* write; with none there is nothing " + "for a rollback to undo and the assertions below prove nothing" + ) + assert await _messages_for_run(db_session, run_id) == [], ( + "the chunks written before the failure are still in agent_messages. _release_foa " + "commits, so they are now permanent: a fragment of a funding post that the retry " + "will duplicate rather than replace" + ) + assert await _claimed_numbers(db_session) == set(), ( + f"{opportunity['number']} is still claimed after the write failed — the FOA is " + "permanently retired: never fully posted, never retried" + ) + + +# --------------------------------------------------------------- the description bug + + +async def test_the_draft_prompt_is_built_from_an_empty_description( + biomedical_candidates, monkeypatch, api_budget, +): + """PINNED BUG, reported and deliberately unfixed — do not "fix" this test green. + + grants.gov `search2` returns no `description` field, so `search_opportunities` maps it + to `""` and grantbot.py:306 interpolates that empty string into the drafting prompt. + The prompt then asks the model to "summarize the scientific scope and goals" of an FOA + it has been told nothing about beyond the title. + + Three halves, so a green run means "verified", not "could not look": + + 1. live: every search2 hit has an empty `description`, while `title` is non-empty — + the control that proves the response itself is not empty; + 2. the real `_draft_post` prompt, captured, carries `Description:` with nothing after it; + 3. the `Synopsis:` line, whose content depends on `fetch_opportunity_detail`. With the + detail backend down (T3's finding, re-checked live here) the model is left with a + title and nothing else; when it recovers, the assertion flips to requiring content. + """ + opportunity = biomedical_candidates[0] + + api_budget.wait("grants") + hits = await grants.search_opportunities("cancer", agencies=["HHS-NIH11"], rows=10) + assert hits, ( + "search2 returned nothing for 'cancer' at HHS-NIH11 — grants.gov is down or the " + "oppHits path moved; the description claim is unchecked either way" + ) + assert all(h["title"].strip() for h in hits), ( + "CONTROL FAILED: live hits came back with empty titles too, so an empty " + "description would just mean the whole response is empty" + ) + with_description = [h["number"] for h in hits if h["description"]] + assert not with_description, ( + f"search2 now returns a description for {with_description} — the bug at " + "grantbot.py:306 (an empty description fed to the drafting LLM) may be gone. " + "Verify and update the reported issue rather than deleting this assertion" + ) + + # Capture the prompt the real _draft_post builds, without spending a token on it. + captured: dict[str, str] = {} + + async def _capture(system_prompt, messages, **kwargs): + captured["system"] = system_prompt + captured["user"] = messages[-1]["content"] + return json.dumps({"channel": "funding-opportunities", "post_text": "captured"}) + + monkeypatch.setattr("src.services.llm.generate_agent_response", _capture) + + api_budget.wait("grants") + detail = await grants.fetch_opportunity_detail(str(opportunity["id"])) + drafted = await grantbot._draft_post(detail or opportunity) + assert drafted is not None and captured, "_draft_post never reached the LLM stage" + + # Slice the prompt on grantbot.py's own literal line prefixes rather than parsing + # it: a description can contain newlines, and a line-wise parse would silently read + # only its first line. + prompt = captured["user"] + for prefix in ("Title: ", "\nNumber: ", "\nAgency: ", "\nClose Date: ", + "\nDescription: ", "\nSynopsis: "): + assert prefix in prompt, ( + f"the drafting prompt no longer contains a {prefix.strip()!r} line — " + f"grantbot._draft_post's opp_text was restructured:\n{prompt[:400]!r}" + ) + description = prompt[ + prompt.index("\nDescription: ") + len("\nDescription: "):prompt.index("\nSynopsis: ") + ] + synopsis = prompt[prompt.index("\nSynopsis: ") + len("\nSynopsis: "):] + assert prompt.splitlines()[0].removeprefix("Title: ").strip(), ( + f"the drafting prompt has no Title either: {prompt[:300]!r}" + ) + + if detail is None: + assert description == "", ( + "the drafting prompt now carries a Description even though " + "fetch_opportunity_detail returned None and search2 supplies none. " + f"Prompt:\n{prompt[:400]!r}" + ) + assert synopsis == "", ( + f"unexpected Synopsis with no detail available: {prompt[:400]!r}" + ) + assert len(captured["user"]) < 400, ( + "PROVIDER DOWN + the description bug together: grants.gov's fetchOpportunity " + "backend is unavailable (T3's finding, still true) and search2 supplies no " + "description, so the entire user prompt behind every funding post GrantBot " + f"writes today is {len(captured['user'])} characters of title, number, agency " + f"and close date:\n{captured['user']!r}\nIf this assertion fails the prompt " + "grew — check whether the detail endpoint recovered" + ) + else: + assert (description + synopsis).strip(), ( + f"fetchOpportunity recovered and returned {sorted(detail)}, but the drafting " + "prompt still has neither a Description nor a Synopsis — the mapping in " + f"services/grants.py is dropping both. Prompt:\n{prompt[:400]!r}" + ) + + +# --------------------------------------------------------------- funding-rules validators + + +def test_the_announcement_detector_only_matches_first_person_openers(): + """CHARACTERIZATION of a real gap, found by the live test below. NOT a fix. + + `_ANNOUNCEMENT_PHRASES` in funding_rules.py is anchored on an explicit first-person + subject — `I'll `, `I will `, `I'm going to `. Slack prose drops + the subject, and every one of the three replies below (verbatim + `claude-sonnet-4-6` output from the live test, 2026-07-30) announces a spin-off and + is NOT flagged. The consequence is the incident the rule was written for: an agent + replies "will spin up a thread", never does, and the funding thread dies with an + announcement instead of a contribution. + + This test needs no network — it is here rather than in tests/unit/ so that the + finding sits next to the live measurement that produced it, and so the paired + controls can be read together. It is a pin, not an aspiration: if someone widens the + phrase list, this test SHOULD fail, and the right response is to delete it. + """ + pairs = [ + # (an equivalent the detector DOES catch, the real reply it MISSES) + ("I'll spin up a dedicated thread for our group on this one later this week.", + "will spin up a dedicated thread for our group on this one later this week"), + ("I'm going to start a separate thread for coordinating our response to this.", + "going to start a separate thread for coordinating our response to this — stay tuned"), + ("I'll post a dedicated thread for this later today once I've reviewed it.", + "will post a dedicated thread for this later today once i've had a chance to review"), + ] + for covered, subject_dropped in pairs: + assert is_announcement_only_funding_reply(covered) is True, ( + "CONTROL FAILED: the first-person form is no longer caught either, so the " + "miss below is not about the dropped subject — the detector is simply off. " + f"Reply: {covered!r}" + ) + assert is_announcement_only_funding_reply(subject_dropped) is False, ( + "the subject-dropped announcement is now caught. The gap this test pins has " + "been closed (good) — delete this test and tighten the live one. " + f"Reply: {subject_dropped!r}" + ) + + +@pytest.mark.real_llm +@needs_llm +async def test_funding_rules_validators_against_real_model_output( + biomedical_candidates, api_budget, +): + """The `funding_rules` validators, judged on prose a real model wrote. + + Every existing test of these regexes feeds them strings their author wrote while + writing the regexes, which cannot show whether they match how a model actually + phrases things. Two real calls: one asking for the non-compliant replies the rules + exist to stop (announcement-only spin-off notices and social acknowledgments), one + asking for compliant substantive replies. + + The two directions carry different weight and are asserted differently: + + - false POSITIVES (a real scientific reply silenced as an announcement or an "ack") + are the damaging direction and the bar is zero; + - false NEGATIVES are a real leak, but the *rate* is model output and would make this + test flap. So the bar here is that each detector catches something (it is alive + against prose it did not author) and that no miss was caused by + `_SUBSTANTIVE_MARKERS_RE` firing on a reply with no science in it — that override + exists to protect contributions, and an override that fires on empty replies is a + worse bug than a phrase list that is merely incomplete. The measured miss rate is + characterized deterministically in + `test_the_announcement_detector_only_matches_first_person_openers`. + + Neither number means anything without the other: a detector returning True for + everything scores perfectly on violations, and one returning False for everything + scores perfectly on compliant replies. + """ + from src.agent.funding_rules import _SUBSTANTIVE_MARKERS_RE + from src.services.llm import generate_agent_response + + settings = grantbot.get_settings() + foa = next( + (o for o in biomedical_candidates + if re.match(r"^(PAR?|RFA)-", o["number"], re.IGNORECASE)), + biomedical_candidates[0], + ) + context = ( + f"A GrantBot funding post in Slack:\n\n:moneybag: *Funding Opportunity*\n" + f"*{foa['title']}*\n{foa['number']} | Closes: {foa['close_date']}\n" + ) + + violations_raw = await generate_agent_response( + system_prompt=( + "You are simulating replies that lab PI agents post in a Slack funding " + "thread. Produce examples of two kinds of reply that the funding-thread " + "rules forbid.\n\n" + "\"announcement_only\": 5 replies that merely ANNOUNCE that the PI will " + "create a dedicated spin-off thread later, instead of contributing. They " + "must contain no scientific content at all — no aims, reagents, models, " + "assays, techniques, targets or mechanisms.\n\n" + "\"acknowledgment_only\": 5 purely social one-liners (thanks, agreement, " + "confirmation). Under 100 characters, no question mark, no scientific " + "content, and do NOT quote the FOA number.\n\n" + "Write the way a terse scientist types in Slack. Respond with ONLY JSON: " + '{"announcement_only": [...], "acknowledgment_only": [...]}' + ), + messages=[{"role": "user", "content": context}], + model=settings.llm_agent_model_sonnet, + max_tokens=900, + log_meta={"agent_id": "grantbot", "phase": "t10-violations"}, + ) + compliant_raw = await generate_agent_response( + system_prompt=( + "You are simulating replies that lab PI agents post in a Slack funding " + "thread. Produce 5 GOOD replies: each states a concrete scientific " + "contribution to a joint application — a specific aim, a reagent, a model " + "system, an assay or a platform the lab owns — in 2 to 3 sentences. Each " + "reply must tag exactly one collaborator, chosen from @WisemanBot, " + "@CravattBot and @PetrascheckBot, written exactly like that. Respond with " + 'ONLY JSON: {"substantive": [...]}' + ), + messages=[{"role": "user", "content": context}], + model=settings.llm_agent_model_sonnet, + # Comfortably above what five 2-3 sentence replies need: a stop_reason of + # max_tokens makes generate_agent_response retry, which is a second billed call. + max_tokens=1500, + log_meta={"agent_id": "grantbot", "phase": "t10-compliant"}, + ) + + def _parse(raw: str, key: str) -> list[str]: + text = raw.strip() + start = text.find("{") + assert start >= 0, ( + f"the model did not return JSON for {key!r}; this test cannot proceed. " + f"Raw:\n{raw[:600]!r}" + ) + # raw_decode stops at the end of the first complete object. A stray trailing + # brace — which this model has produced here — breaks a find/rfind slice. + try: + payload, _ = json.JSONDecoder().raw_decode(text[start:]) + except json.JSONDecodeError as exc: + pytest.fail( + f"the model's {key!r} response is not parseable JSON ({exc}). This is a " + f"harness problem, not a funding_rules result. Raw:\n{raw[:600]!r}" + ) + items = payload.get(key) or [] + assert len(items) >= 4, ( + f"the model returned {len(items)} {key!r} examples, too few to measure " + f"against: {items}" + ) + return [str(i) for i in items] + + announcements = _parse(violations_raw, "announcement_only") + acks = _parse(violations_raw, "acknowledgment_only") + substantive = _parse(compliant_raw, "substantive") + + caught_ann = [t for t in announcements if is_announcement_only_funding_reply(t)] + missed_ann = [t for t in announcements if t not in caught_ann] + assert caught_ann, ( + f"is_announcement_only_funding_reply caught NONE of {len(announcements)} " + "announcement-only replies a real model wrote. Its phrase list no longer " + "overlaps how a model phrases a spin-off announcement at all, and the atomic " + "spin-off rule is unenforced. Replies:\n " + + "\n ".join(repr(t) for t in announcements) + ) + caught_ack = [t for t in acks if is_acknowledgment_only_funding_reply(t)] + missed_ack = [t for t in acks if t not in caught_ack] + assert caught_ack, ( + f"is_acknowledgment_only_funding_reply caught NONE of {len(acks)} " + "acknowledgment-only replies. Replies:\n " + + "\n ".join(repr(t) for t in acks) + ) + for label, missed in (("announcement", missed_ann), ("acknowledgment", missed_ack)): + for text in missed: + marker = _SUBSTANTIVE_MARKERS_RE.search(text) + assert marker is None, ( + f"an {label}-only reply with no scientific content was let through " + f"because _SUBSTANTIVE_MARKERS_RE matched {marker.group(0)!r}. That " + "override exists to stop the filters suppressing real contributions; " + "firing on an empty reply means it is too broad and every violation " + f"containing that word is now invisible. Reply: {text!r}" + ) + + false_ann = [t for t in substantive if is_announcement_only_funding_reply(t)] + assert not false_ann, ( + "a substantive reply was classified as announcement-only and would have been " + "suppressed — the damaging direction, because it silences a real scientific " + "contribution:\n " + "\n ".join(repr(t) for t in false_ann) + ) + false_ack = [t for t in substantive if is_acknowledgment_only_funding_reply(t)] + assert not false_ack, ( + "a substantive reply was classified as acknowledgment-only:\n " + + "\n ".join(repr(t) for t in false_ack) + ) + + # The summarizer, over the same real replies. + log = MessageLog() + root_ts = "1700000000.000001" + log.append(LogEntry( + ts=root_ts, channel="funding-opportunities", sender_agent_id=None, + sender_name="GrantBot", content=context, thread_ts=None, + posted_at=float(root_ts), is_bot=True, + )) + for index, text in enumerate(substantive, start=2): + ts = f"1700000000.{index:06d}" + log.append(LogEntry( + ts=ts, channel="funding-opportunities", sender_agent_id="wiseman", + sender_name="WisemanBot", content=text, thread_ts=root_ts, + posted_at=float(ts), is_bot=True, + )) + + summary = summarize_funding_thread(log, root_ts) + assert len(summary.alignments) == len(substantive), ( + f"summarize_funding_thread recorded {len(summary.alignments)} alignments for " + f"{len(substantive)} replies — a late joiner would be shown an incomplete thread" + ) + # All replies share one sender, so the summarizer dedups pairings to one per tagged + # bot. Compare against the tags actually present rather than against the three the + # prompt offered — the model chooses which to use. + tagged_bots = { + m.group(1).lower() for t in substantive for m in re.finditer(r"@(\w+[Bb]ot)\b", t) + } + assert tagged_bots, ( + "the model tagged no collaborator in any of its replies, so the pairing half of " + f"summarize_funding_thread is untested this run. Replies: {substantive}" + ) + assert {b.lower() for _, b in summary.pairings_proposed} == tagged_bots, ( + f"the replies tag {sorted(tagged_bots)} but summarize_funding_thread reports " + f"{sorted(b.lower() for _, b in summary.pairings_proposed)} — proposed " + "collaborations are being lost from the summary a late joiner is shown" + ) diff --git a/tests/integration/test_harness_smoke.py b/tests/integration/test_harness_smoke.py index 380a113..1b26482 100644 --- a/tests/integration/test_harness_smoke.py +++ b/tests/integration/test_harness_smoke.py @@ -7,7 +7,12 @@ async def test_container_is_migrated(engine): async with engine.connect() as conn: v = (await conn.execute(text("SELECT version_num FROM alembic_version"))).scalar_one() - assert v == "0018" + # Head-revision pin: bump it deliberately with each new migration. This is + # the guard that catches a branch whose migration was renumbered late — see + # .notes/cohort-system-v2.md §14 for what a duplicate revision id costs. + # 0019-0021 db-primary-conversations, 0022 cohorts, + # 0023 researcher_profiles synthesis provenance, 0024 agents.role column + assert v == "0024" async def test_writes_are_rolled_back_part1(db_session): diff --git a/tests/integration/test_message_persistence.py b/tests/integration/test_message_persistence.py new file mode 100644 index 0000000..2ec661a --- /dev/null +++ b/tests/integration/test_message_persistence.py @@ -0,0 +1,779 @@ +"""Integration tests for the DB-primary message persistence guards (PR #19 review). + +Exercised against the real migrated Postgres so the actual ON CONFLICT upsert +(including its M1a human-row guard) is validated, not just the Python logic. +See specs/local-db-conversations.md. +""" + +import time +from datetime import timedelta + +import pytest +from sqlalchemy import func, select + +from src.agent.agent import Agent +from src.agent.message_log import LogEntry +from src.agent.simulation import ( + PI_INBOX_LOOKBACK_S, + REBUILD_WINDOW_S, + SimulationEngine, +) +from src.models import AgentMessage, PiDmMessage +from tests import factories + +pytestmark = pytest.mark.integration + + +class _FixtureSessionFactory: + """Route the engine's self-opened session at the rolled-back test session. + + _flush_persisted does ``async with self.session_factory() as db: ... await + db.commit()``. The test session runs in create_savepoint mode, so commit() + just releases a savepoint and the outer transaction still rolls back at + teardown. __aexit__ must NOT close the fixture-owned session. + """ + + def __init__(self, session): + self._s = session + + def __call__(self): + return self + + async def __aenter__(self): + return self._s + + async def __aexit__(self, *exc): + return False + + +def _engine_for(session, run_id, agents=None): + return SimulationEngine( + agents=agents or [], slack_clients={}, + session_factory=_FixtureSessionFactory(session), + simulation_run_id=run_id, + ) + + +class _RecordingPiHandler: + """Minimal PIHandler stand-in that records handle_dm calls.""" + + def __init__(self): + self.calls = [] + + async def handle_dm(self, agent_id, pi_user_id, content): + self.calls.append((agent_id, pi_user_id, content)) + + +async def test_flush_upsert_does_not_clobber_human_row_with_bot(db_session): + # M1a: a cross-process canonical-id collision must not let a bot message + # overwrite an existing human (PI) row in the now-authoritative store. + run = await factories.make_simulation_run(db_session) + collide_ts = "1700000000.123456" + await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=False, + channel_id="local:general", channel_name="general", + message_ts=collide_ts, posted_at=float(collide_ts), + content="HUMAN PI MESSAGE", sender_name="Dr Human (PI)", + ) + + engine = _engine_for(db_session, run.id) + engine._pending_persist = [LogEntry( + ts=collide_ts, channel="general", sender_agent_id="subot", + sender_name="SuBot", content="BOT CLOBBER ATTEMPT", + posted_at=float(collide_ts), is_bot=True, + )] + await engine._flush_persisted() + + row = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.message_ts == collide_ts, + ))).scalar_one() + assert row.is_bot is False + assert row.agent_id is None + assert row.content == "HUMAN PI MESSAGE" + + +async def test_flush_upsert_still_updates_own_bot_row(db_session): + # The guard must not break the legitimate idempotent re-flush / slack-mirror + # path: a bot row re-flushed at the same ts updates in place. + run = await factories.make_simulation_run(db_session) + bot_ts = "1700000000.222222" + engine = _engine_for(db_session, run.id) + for text in ("v1", "v2-updated"): + engine._pending_persist = [LogEntry( + ts=bot_ts, channel="general", sender_agent_id="subot", + sender_name="SuBot", content=text, + posted_at=float(bot_ts), is_bot=True, + )] + await engine._flush_persisted() + + row = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.message_ts == bot_ts, + ))).scalar_one() + assert row.content == "v2-updated" + + +async def test_flush_upsert_allows_human_reflush(db_session): + # An ingested human PI message re-flushed by the engine (is_bot=False both + # sides) must still update — the guard only blocks bot-over-human. + run = await factories.make_simulation_run(db_session) + ts = "1700000000.333333" + await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=False, + channel_id="local:general", channel_name="general", + message_ts=ts, posted_at=float(ts), + content="original", sender_name="PI", + ) + engine = _engine_for(db_session, run.id) + engine._pending_persist = [LogEntry( + ts=ts, channel="general", sender_agent_id=None, + sender_name="PI", content="edited", posted_at=float(ts), is_bot=False, + )] + await engine._flush_persisted() + + row = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.message_ts == ts, + ))).scalar_one() + assert row.content == "edited" + assert row.is_bot is False + + +# --------------------------------------------------------------- +# R1 — the collision the M1a guard resolves lossily must not be reachable in the +# first place: concurrent writers mint into disjoint slots, so both messages +# survive against the real unique constraint. +# --------------------------------------------------------------- + +async def test_concurrent_writers_both_persist_at_the_same_instant(db_session, monkeypatch): + import time as time_mod + + from src.agent.ids import ( + WRITER_ENGINE, + WRITER_WEB, + TsMinter, + set_default_writer_id, + ) + from src.services.pi_inbox import record_pi_message + + run = await factories.make_simulation_run(db_session) + + # Freeze the clock: every mint in this test sees the identical microsecond, + # which is exactly the case that used to yield one id and drop a message. + monkeypatch.setattr(time_mod, "time_ns", lambda: 1_800_000_000_000_000_000) + + engine = _engine_for(db_session, run.id) + engine._ts_minter = TsMinter(WRITER_ENGINE) + set_default_writer_id(WRITER_WEB) + + bot_ts = engine.mint_ts() + engine._pending_persist = [LogEntry( + ts=bot_ts, channel="general", sender_agent_id="subot", + sender_name="SuBot", content="BOT MESSAGE", + posted_at=float(bot_ts), is_bot=True, + )] + await engine._flush_persisted() + + # The web app's writer, minting at the same frozen instant. + pi_msg = await record_pi_message( + db_session, run_id=run.id, channel_name="general", + content="PI: please pivot to aging biology", sender_name="Dr Human (PI)", + ) + await db_session.flush() + + assert pi_msg.message_ts != bot_ts + rows = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + ))).scalars().all() + contents = {r.content for r in rows} + assert contents == {"BOT MESSAGE", "PI: please pivot to aging biology"} + + +# --------------------------------------------------------------- +# H2 — the inbox pollers must not skip a row that committed below the cursor +# (the stamp is written at row creation, so a late-committing PI row lands below +# a cursor already advanced past it). +# --------------------------------------------------------------- + +async def test_inbound_poller_ingests_pi_row_committed_below_cursor(db_session): + run = await factories.make_simulation_run(db_session) + engine = _engine_for(db_session, run.id) + # A PI row whose stamp is *below* the cursor but within the lookback window — + # the H2 race. The old `> cursor` filter skipped it forever. + below_ts = "1700000150.000000" + row = await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=False, + channel_id="local:general", channel_name="general", + message_ts=below_ts, posted_at=float(below_ts), + content="late-committed PI message", sender_name="PI", + ) + await db_session.refresh(row) + # The cursor has already advanced (engine flushed its own later message). + # Derived from the row's own created_at so the assertion doesn't depend on + # this process's clock matching the DB server's — the point of R3. + engine._pi_inbox_cursor = row.created_at + timedelta(seconds=50) + + await engine._poll_inbound_from_db() + + entry = engine.message_log.get_entry(below_ts) + assert entry is not None + assert entry.content == "late-committed PI message" + + +async def test_inbound_poller_skips_row_older_than_lookback(db_session): + # Bounds the re-scan: a row far below the lookback floor is not re-queried. + run = await factories.make_simulation_run(db_session) + engine = _engine_for(db_session, run.id) + ancient_ts = "1700000000.000000" + row = await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=False, + channel_id="local:general", channel_name="general", + message_ts=ancient_ts, posted_at=float(ancient_ts), + content="ancient", sender_name="PI", + ) + await db_session.refresh(row) + engine._pi_inbox_cursor = row.created_at + timedelta( + seconds=PI_INBOX_LOOKBACK_S + 100 + ) + await engine._poll_inbound_from_db() + assert engine.message_log.get_entry(ancient_ts) is None + + +async def test_inbound_poller_delivers_a_row_from_a_skewed_writer_clock(db_session): + # R3: a writer whose clock is far behind the engine's stamps posted_at well + # below the cursor. Paging over created_at (the DB server's clock) delivers + # it anyway; the old posted_at cursor dropped it silently and forever. + run = await factories.make_simulation_run(db_session) + engine = _engine_for(db_session, run.id) + + recent = await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="1700009000.000000", posted_at=1700009000.0, + content="engine post", sender_name="SuBot", + ) + await db_session.refresh(recent) + engine._pi_inbox_cursor = recent.created_at + + skewed_ts = "1600000000.000000" # ~3 years of clock skew + skewed = await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=False, + channel_id="local:general", channel_name="general", + message_ts=skewed_ts, posted_at=float(skewed_ts), + content="PI message from a skewed host", sender_name="PI", + ) + await db_session.refresh(skewed) + assert skewed.posted_at < engine._pi_inbox_cursor.timestamp() - PI_INBOX_LOOKBACK_S + + await engine._poll_inbound_from_db() + + entry = engine.message_log.get_entry(skewed_ts) + assert entry is not None + assert entry.content == "PI message from a skewed host" + + +async def test_dm_poller_ingests_below_cursor_then_dedups(db_session): + run = await factories.make_simulation_run(db_session) + agent = Agent("su", "SuBot", "Andrew Su") + engine = _engine_for(db_session, run.id, agents=[agent]) + handler = _RecordingPiHandler() + engine._pi_handler = handler + + below_ts = "1700000150.000000" + dm = PiDmMessage( + simulation_run_id=run.id, agent_id="su", pi_user_id="local:x", + direction="inbound", content="standing instruction", + sender_name="PI", ts=below_ts, posted_at=float(below_ts), + ) + db_session.add(dm) + await db_session.flush() + await db_session.refresh(dm) + engine._pi_dm_cursor = dm.created_at + timedelta(seconds=50) + + # First poll ingests the below-cursor row (H2)... + await engine._poll_pi_dms_from_db() + assert handler.calls == [("su", "local:x", "standing instruction")] + + # ...and the lookback re-scan on the next poll does NOT re-process it. + await engine._poll_pi_dms_from_db() + assert len(handler.calls) == 1 + + +async def test_seed_pi_dm_cursor_prevents_replay_on_restart(db_session): + # Seeding the seen-set (not just the cursor) means the first poll's lookback + # re-scan doesn't replay recent history through handle_dm after a restart. + run = await factories.make_simulation_run(db_session) + ts = "1700000150.000000" + db_session.add(PiDmMessage( + simulation_run_id=run.id, agent_id="su", pi_user_id="local:x", + direction="inbound", content="old directive", + sender_name="PI", ts=ts, posted_at=float(ts), + )) + await db_session.flush() + + agent = Agent("su", "SuBot", "Andrew Su") + engine = _engine_for(db_session, run.id, agents=[agent]) + handler = _RecordingPiHandler() + engine._pi_handler = handler + + await engine._seed_pi_dm_cursor() + assert ts in engine._pi_dm_seen + await engine._poll_pi_dms_from_db() + assert handler.calls == [] + + +# --------------------------------------------------------------- +# B1 — the cosmetic run-stats COUNT is throttled, not run every flush. +# --------------------------------------------------------------- + +async def _count_messages(session, run_id): + return (await session.execute( + select(func.count(AgentMessage.id)).where( + AgentMessage.simulation_run_id == run_id + ) + )).scalar_one() + + +async def test_flush_throttles_run_stats_count(db_session): + run = await factories.make_simulation_run(db_session) + engine = _engine_for(db_session, run.id) + + def _enqueue(ts, content): + engine._pending_persist = [LogEntry( + ts=ts, channel="general", sender_agent_id="su", + sender_name="SuBot", content=content, posted_at=float(ts), is_bot=True, + )] + + # First flush refreshes the counter. + _enqueue("100000001.000000", "a") + await engine._flush_persisted() + assert run.total_messages == 1 + + # Second flush within the interval inserts a row but does NOT recount — the + # counter is intentionally stale (throttled) even though 2 rows now exist. + _enqueue("100000002.000000", "b") + await engine._flush_persisted() + assert await _count_messages(db_session, run.id) == 2 + assert run.total_messages == 1 + + # force_stats (used on shutdown) recomputes immediately. + _enqueue("100000003.000000", "c") + await engine._flush_persisted(force_stats=True) + assert run.total_messages == 3 + + +# --------------------------------------------------------------- +# B2 — the startup rebuild loads a bounded window (recent + undecided threads), +# and old closed threads are hydrated on demand. +# --------------------------------------------------------------- + +async def test_rebuild_windows_recent_and_undecided_only(db_session): + run = await factories.make_simulation_run(db_session) + old = time.time() - REBUILD_WINDOW_S - 100_000 + now = time.time() + + # Old + closed (has a ThreadDecision) → windowed out. + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="OLDCLOSED", thread_ts=None, posted_at=old, content="old closed root", + ) + await factories.make_thread_decision(db_session, run=run, thread_id="OLDCLOSED") + + # Old + undecided (no ThreadDecision) → loaded in full. + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="OLDLIVE", thread_ts=None, posted_at=old, content="old live root", + ) + # Recent → loaded. + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="RECENT", thread_ts=None, posted_at=now, content="recent", + ) + + engine = _engine_for(db_session, run.id) + await engine._rebuild_state_from_db() + + assert engine.message_log.get_entry("OLDCLOSED") is None + assert engine.message_log.get_entry("OLDLIVE") is not None + assert engine.message_log.get_entry("RECENT") is not None + + +async def test_hydrate_thread_loads_windowed_out_thread(db_session): + run = await factories.make_simulation_run(db_session) + old = time.time() - REBUILD_WINDOW_S - 100_000 + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="THR", thread_ts=None, posted_at=old, content="root", + ) + await factories.make_agent_message( + db_session, run=run, agent_id="lairson", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="THR-r1", thread_ts="THR", posted_at=old + 1, content="reply", + ) + + engine = _engine_for(db_session, run.id) + assert engine.message_log.get_entry("THR") is None # not yet loaded + + await engine._hydrate_thread_from_db("THR") + assert engine.message_log.get_entry("THR") is not None + assert len(engine.message_log.get_thread_history("THR")) == 2 + + # Idempotent — a second hydrate doesn't duplicate. + await engine._hydrate_thread_from_db("THR") + assert len(engine.message_log.get_thread_history("THR")) == 2 + + +# --------------------------------------------------------------- +# The Slack mirror mapping has to survive a restart, otherwise the engine +# cannot tell a Slack-backed thread from a DB-origin one and would mirror +# replies against an id Slack has never seen. +# --------------------------------------------------------------- + +async def test_rebuild_restores_the_slack_mapping(db_session): + run = await factories.make_simulation_run(db_session) + now = time.time() + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="C0SLACK", channel_name="general", + message_ts="MIRRORED", posted_at=now, content="db-origin, then mirrored", + slack_ts="1700009999.111111", slack_channel_id="C0SLACK", + ) + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="DBONLY", posted_at=now, content="never mirrored", + ) + + engine = _engine_for(db_session, run.id) + await engine._rebuild_state_from_db() + + assert engine.message_log.get_entry("MIRRORED").slack_ts == "1700009999.111111" + assert engine._slack_parent_ts("MIRRORED") == "1700009999.111111" + # A DB-origin root has no Slack presence — replies to it must not be mirrored. + assert engine.message_log.get_entry("DBONLY").slack_ts is None + assert engine._slack_parent_ts("DBONLY") is None + + +async def test_rebuild_never_infers_a_slack_ts_from_the_channel_id(db_session): + """A NULL slack_ts means "not on Slack", even in a real Slack channel. + + The rebuild used to infer the mapping for such a row, on the theory that it + predated Stage 6. But a DB-origin message can carry a real Slack channel id + too — a PI message written through the web inbox resolves channel_id from the + agent_channels row, and so does an agent post whose mirror failed. Inferring + hands _slack_parent_ts a canonical id Slack never issued, which then goes out + as a chat.postMessage thread_ts and orphans the reply. Legacy rows are + repaired by scripts/backfill_slack_ts.py, which asks Slack instead of guessing. + """ + run = await factories.make_simulation_run(db_session) + # Exactly the shape that used to be mis-inferred: web-written PI message, + # locally-minted canonical id, stored against the channel's real Slack id. + await factories.make_agent_message( + db_session, run=run, agent_id=None, is_bot=False, + channel_id="C0SLACK", channel_name="general", + message_ts="1800000000.000001", posted_at=1800000000.000001, + content="@SuBot a PI message written from the web", slack_ts=None, + sender_name="Dr Human (PI)", + ) + + engine = _engine_for(db_session, run.id) + await engine._rebuild_state_from_db() + + assert engine.message_log.get_entry("1800000000.000001").slack_ts is None + assert engine._slack_parent_ts("1800000000.000001") is None + + +# --------------------------------------------------------------- +# R1 (residual) — every canonical id must come from the shared minter, so it +# carries its process's writer slot. The Slack-off private-channel handover was +# the last site formatting ids straight off time.time(). +# --------------------------------------------------------------- + + +async def test_offline_migration_mints_ids_in_its_own_writer_slot(db_session, monkeypatch): + import time as time_mod + + from src.agent.ids import ( + WRITER_ENGINE, + WRITER_SLOT_MODULUS, + WRITER_WEB, + TsMinter, + set_default_writer_id, + ) + from src.services.private_channels import _migrate_offline + + run = await factories.make_simulation_run(db_session) + pi_user = await factories.make_user(db_session) + td = await factories.make_thread_decision( + db_session, run=run, agent_a="su", agent_b="wiseman", + channel="general", summary_text="A joint proposal.", + ) + + # Freeze BOTH clocks the two id schemes read (time_ns for the minter, + # time for the old hand-rolled format), so the engine and the migration mint + # at the identical microsecond — the case that used to collide. + monkeypatch.setattr(time_mod, "time_ns", lambda: 1_800_000_000_000_000_000) + monkeypatch.setattr(time_mod, "time", lambda: 1_800_000_000.0) + + engine = _engine_for(db_session, run.id) + engine._ts_minter = TsMinter(WRITER_ENGINE) + set_default_writer_id(WRITER_WEB) + + bot_ts = engine.mint_ts() + engine._pending_persist = [LogEntry( + ts=bot_ts, channel="general", sender_agent_id="su", + sender_name="SuBot", content="BOT MESSAGE", + posted_at=float(bot_ts), is_bot=True, + )] + await engine._flush_persisted() + + # The web process writes the handover at that same frozen instant. Under the + # old scheme its first id was f"{time.time():.6f}" == the engine's id, so the + # ORM insert below hit uq_agent_messages_run_ts. + await _migrate_offline( + db_session, + thread_decision=td, + creator_agent_id="su", + creator_pi_user=pi_user, + guidance_text="Narrow the aim to one assay.", + a="su", b="wiseman", + other_agent_id="wiseman", + origin_channel_name="general", + ) + await db_session.flush() + + rows = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + ))).scalars().all() + assert "BOT MESSAGE" in {r.content for r in rows} + + handover = [r for r in rows if r.message_ts != bot_ts] + # 2+ handover posts in the new private channel, plus the origin-thread marker. + assert len(handover) >= 3 + assert any(r.thread_ts == td.thread_id for r in handover) + + # Every handover id sits in the web writer's residue class, so it can never + # coincide with an engine- or GrantBot-minted id ... + for r in handover: + assert int(r.message_ts.partition(".")[2]) % WRITER_SLOT_MODULUS == WRITER_WEB + # ... and they stay distinct and float-ordered (posted_at == float(ts)). + minted = sorted(r.message_ts for r in handover) + assert len(set(minted)) == len(minted) + floats = [float(t) for t in minted] + assert all(b > a for a, b in zip(floats, floats[1:], strict=False)) + assert all(r.posted_at == float(r.message_ts) for r in handover) + + +# --------------------------------------------------------------- +# The Slack-*on* migration used to post the handover to Slack without recording +# it in agent_messages — the last place a message existed on Slack before it +# existed in the primary store. +# --------------------------------------------------------------- + + +def _patch_slack_migration(monkeypatch, clients: dict): + """Route private_channels' Slack surface at FakeSlackClient instances.""" + from src.services import private_channels as pc + from tests.fakes import FakeSlackClient + + async def _enabled(*args, **kwargs): + return True + + async def _token(db, agent_id): + return f"xoxb-fake-{agent_id}" + + async def _other_pi(db, agent_id): + return None, None # no claimed PI on the other side — skips the DM branch + + def _client(agent_id, token): + return clients.setdefault(agent_id, FakeSlackClient(agent_id=agent_id)) + + monkeypatch.setattr(pc, "_slack_enabled_for_migration", _enabled) + monkeypatch.setattr(pc, "_get_or_fail_bot_token", _token) + monkeypatch.setattr(pc, "_resolve_other_pi", _other_pi) + monkeypatch.setattr(pc, "_make_client", _client) + return pc + + +async def test_slack_migration_mirrors_the_handover_into_the_db(db_session, monkeypatch): + clients: dict = {} + pc = _patch_slack_migration(monkeypatch, clients) + + run = await factories.make_simulation_run(db_session) + pi_user = await factories.make_user(db_session) + # A Slack-born origin root: stored against a real Slack channel, so its + # canonical id is also its Slack ts. + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="C0ORIGIN", channel_name="general", + message_ts="1700000000.000500", posted_at=1700000000.0005, + content="origin root", slack_ts="1700000000.000500", + ) + td = await factories.make_thread_decision( + db_session, run=run, agent_a="su", agent_b="wiseman", + channel="general", thread_id="1700000000.000500", + summary_text="A joint proposal.", + ) + + result = await pc.migrate_public_thread_to_private( + db_session, thread_decision=td, creator_agent_id="su", + creator_pi_user=pi_user, guidance_text="Narrow the aim to one assay.", + ) + await db_session.flush() + + rows = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.content != "origin root", + ))).scalars().all() + + # Sorted by canonical id, which is post order here (the fake ts increments). + private_rows = sorted( + (r for r in rows if r.channel_name == result.channel_name), + key=lambda r: r.message_ts, + ) + close_rows = [r for r in rows if r.channel_name == "general"] + assert len(private_rows) >= 2 # the handover posts + assert len(close_rows) == 1 # the origin-thread close marker + + # Slack-on parity (design rule 1): the canonical id IS the Slack ts, and the + # mirror mapping is recorded so a later reconcile dedups instead of duplicating. + posted_ts = {p["ts"] for p in clients["su"].posted} + for r in private_rows + close_rows: + assert r.slack_ts == r.message_ts + assert r.message_ts in posted_ts + assert r.posted_at == float(r.message_ts) + assert r.is_bot is True + assert r.sender_name == "suBot" + assert all(r.visibility == "collab_private" for r in private_rows) + # Stored content is the handover text itself (pre-mrkdwn), not a placeholder. + expected = pc._build_handover_messages( + creator_pi_name=pi_user.name, + proposal_summary="A joint proposal.", + guidance_text="Narrow the aim to one assay.", + origin_channel_name="general", + ) + assert [r.content for r in private_rows] == expected + assert any("one assay" in r.content for r in private_rows) + + # The close marker threads on the root's Slack ts, in the origin channel, and + # carries no PI guidance text. + marker = close_rows[0] + assert marker.visibility == "public" + assert marker.thread_ts == "1700000000.000500" + assert marker.slack_thread_ts == "1700000000.000500" + assert "one assay" not in marker.content + # ... and that is what Slack was actually asked to thread on. + threaded = [p for p in clients["su"].posted if p["thread_ts"]] + assert [p["thread_ts"] for p in threaded] == ["1700000000.000500"] + + +async def test_slack_migration_keeps_the_close_marker_db_only_for_a_db_origin_root( + db_session, monkeypatch, +): + """A thread started Slack-off has a minted root id Slack has never seen. + + The marker must not be posted against it (that detaches or errors), but it + still has to land in the DB — the store the simulation actually reads. + """ + clients: dict = {} + pc = _patch_slack_migration(monkeypatch, clients) + + run = await factories.make_simulation_run(db_session) + pi_user = await factories.make_user(db_session) + await factories.make_agent_message( + db_session, run=run, agent_id="su", is_bot=True, + channel_id="local:general", channel_name="general", + message_ts="1800000000.000100", posted_at=1800000000.0001, + content="db-origin root", slack_ts=None, + ) + td = await factories.make_thread_decision( + db_session, run=run, agent_a="su", agent_b="wiseman", + channel="general", thread_id="1800000000.000100", + ) + + await pc.migrate_public_thread_to_private( + db_session, thread_decision=td, creator_agent_id="su", + creator_pi_user=pi_user, guidance_text="Keep going.", + ) + await db_session.flush() + + # Nothing was posted into a thread on Slack ... + assert [p for p in clients["su"].posted if p["thread_ts"]] == [] + # ... but the marker exists in the DB, unmirrored, on the canonical thread. + marker = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.thread_ts == "1800000000.000100", + ))).scalars().one() + assert marker.slack_ts is None + assert marker.slack_thread_ts is None + assert marker.channel_name == "general" + + +# --------------------------------------------------------------- +# The channel poller's bot branch dropped the Slack mirror mapping, so a thread +# rooted at a polled bot post (GrantBot's funding posts) looked DB-origin and +# every reply to it was kept off Slack. +# --------------------------------------------------------------- + + +class _HistoryClient: + """Connected transport that returns one canned bot message from history.""" + + def __init__(self, messages): + self.agent_id = "su" + self._messages = messages + + @property + def is_connected(self): + return True + + def is_bot_user(self, user_id): + return False + + def poll_channel_messages(self, channel_id, oldest="0", limit=100): + return list(self._messages) + + def resolve_user_name(self, user_id): + return user_id + + +async def test_polled_bot_message_keeps_its_slack_mapping(db_session): + run = await factories.make_simulation_run(db_session) + client = _HistoryClient([{ + "ts": "1700000123.456789", + "bot_id": "B0GRANT", + "username": "GrantBot", + "text": ":moneybag: *Funding Opportunity* R01 something", + }]) + + engine = _engine_for(db_session, run.id) + engine.slack_clients = {"su": client} + engine._channel_id_map = {"funding-opportunities": "C0FUNDING"} + engine._channel_visibility = {"funding-opportunities": "public"} + # start() registers this; the poller's append has to reach the DB buffer. + engine.message_log.set_persist_callback(engine._enqueue_persist) + + await engine._poll_slack_for_pi_messages() + + entry = engine.message_log.get_entry("1700000123.456789") + assert entry is not None + # The mapping is what makes a reply mirrorable: without it _slack_parent_ts + # reports "no Slack root" and _post_message keeps the reply DB-only. + assert entry.slack_ts == "1700000123.456789" + assert entry.slack_channel_id == "C0FUNDING" + assert engine._slack_parent_ts("1700000123.456789") == "1700000123.456789" + + # And it survives the flush into the primary store. + await engine._flush_persisted() + row = (await db_session.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run.id, + AgentMessage.message_ts == "1700000123.456789", + ))).scalars().one() + assert row.slack_ts == "1700000123.456789" + assert row.slack_channel_id == "C0FUNDING" + assert row.is_bot is True diff --git a/tests/integration/test_onboarding_flow.py b/tests/integration/test_onboarding_flow.py new file mode 100644 index 0000000..d8f78c5 --- /dev/null +++ b/tests/integration/test_onboarding_flow.py @@ -0,0 +1,1524 @@ +"""Task 7 — the first-run experience: onboarding, profile and settings. + +Fifteen HTTP endpoints across ``src/routers/onboarding.py`` (5), +``src/routers/profile.py`` (6) and ``src/routers/settings.py`` (4) had no direct +coverage, and ``src/services/profile_export.py`` had no test referencing it at all. + +(It was seventeen until ``POST /onboarding/complete`` and ``GET /onboarding/done`` +were deleted as an unreachable duplicate of the terminal step — see +``test_the_terminal_step_*`` below, which inherited their controls.) + +Real ASGI requests, real Postgres, real Jinja templates, real ``profile_export``. +Nothing external runs: the ORCID and Anthropic entry points are replaced with +raising stubs (a first-run route that reached for one would fail loudly rather +than quietly make a network call), SES is a recorder, and the two export +directories are redirected into ``tmp_path`` so the suite never writes into +``profiles/``. + +Discipline (see ``.notes/full-system-test-plan.md``): every absence assertion +carries a positive control in the same test. "The victim's row did not change" +is worthless next to a route that changes nothing for anybody, so each negative +is paired with the same request producing the effect it is supposed to produce. +""" + +import base64 +import json +import re +from collections.abc import Callable +from dataclasses import dataclass +from types import SimpleNamespace + +import pytest +from itsdangerous import TimestampSigner, URLSafeTimedSerializer +from sqlalchemy import func, select + +from src.config import get_settings +from src.models import ( + EmailEngagementTracker, + EmailNotificationPreference, + Job, + ProfileRevision, + Publication, + ResearcherProfile, + User, +) +from src.routers import onboarding as onboarding_router +from src.routers import profile as profile_router +from src.routers import settings as settings_router +from src.services import profile_export +from src.services.email_notifications import _generate_unsubscribe_token +from tests import factories + +pytestmark = pytest.mark.integration + + +# --------------------------------------------------------------------------- +# harness +# --------------------------------------------------------------------------- + + +def _auth(user_id) -> dict: + """Forge the signed session cookie SessionMiddleware would issue.""" + signer = TimestampSigner(get_settings().secret_key) + data = base64.b64encode(json.dumps({"user_id": str(user_id)}).encode()) + return {"Cookie": f"copi-session={signer.sign(data).decode()}"} + + +def _auth_as(user_id, impersonate_id) -> dict: + """Session for ``user_id`` plus the copi-impersonate cookie pointed at another user. + + src/dependencies.get_current_user honours that cookie *only* when the session + user is an admin. It is the one handle any of these 17 endpoints gives a + caller on somebody else's identity, so it is the vector the sweep attacks. + """ + signer = TimestampSigner(get_settings().secret_key) + data = base64.b64encode(json.dumps({"user_id": str(user_id)}).encode()) + return { + "Cookie": ( + f"copi-session={signer.sign(data).decode()}; " + f"copi-impersonate={impersonate_id}" + ) + } + + +@pytest.fixture(autouse=True) +def export_dirs(tmp_path, monkeypatch): + """Redirect both export directories so no test writes into the repo's profiles/.""" + pub, priv = tmp_path / "public", tmp_path / "private" + monkeypatch.setattr(profile_export, "PROFILES_DIR", pub) + monkeypatch.setattr(profile_export, "PRIVATE_PROFILES_DIR", priv) + # onboarding.py bound PRIVATE_PROFILES_DIR into its own namespace at import + # time (the on-disk fallback in the private-profile editor), so patching the + # service module alone would leave that read pointed at the repo. + monkeypatch.setattr(onboarding_router, "PRIVATE_PROFILES_DIR", priv) + return SimpleNamespace(public=pub, private=priv) + + +@pytest.fixture(autouse=True) +def welcome_emails(monkeypatch): + """Recording double for the one SES call these routers make.""" + sent: list[dict] = [] + + import src.services.email as email_mod + + def _record(to_email, name=None, *, user_id=None, force=False): + sent.append({"to": to_email, "name": name, "user_id": user_id}) + return True + + monkeypatch.setattr(email_mod, "send_welcome_email", _record) + return sent + + +@pytest.fixture(autouse=True) +def no_external_calls(monkeypatch): + """ORCID and Anthropic must never be reached from a first-run route. + + Profile generation is the worker's job; these routes only enqueue it. A stub + that raises turns "we accidentally made a network call in a request handler" + into a test failure instead of a slow, flaky, billable test. + """ + + def _boom(label): + async def _f(*_a, **_k): + raise AssertionError(f"{label} was called from a first-run HTTP route") + + return _f + + for fn in ( + "fetch_orcid_record", + "fetch_orcid_profile", + "fetch_orcid_grants", + "fetch_orcid_works", + ): + monkeypatch.setattr(f"src.services.orcid.{fn}", _boom(f"orcid.{fn}")) + for fn in ("synthesize_profile", "synthesize_private_profile", "generate_agent_response"): + monkeypatch.setattr(f"src.services.llm.{fn}", _boom(f"llm.{fn}")) + + +# --- fresh reads ----------------------------------------------------------- +# Column selects rather than ORM loads: the routes commit on the very session +# the test holds, so an already-loaded ORM instance can be stale while a column +# select always shows what is actually in the row. + + +async def _flag(db, uid) -> bool: + return ( + await db.execute(select(User.onboarding_complete).where(User.id == uid)) + ).scalar_one() + + +async def _user_row(db, uid): + return ( + await db.execute( + select( + User.name, + User.email, + User.institution, + User.department, + User.onboarding_complete, + User.email_notification_frequency, + User.email_notifications_paused_by_system, + ).where(User.id == uid) + ) + ).mappings().first() + + +async def _prof(db, uid): + return ( + await db.execute( + select( + ResearcherProfile.research_summary, + ResearcherProfile.techniques, + ResearcherProfile.experimental_models, + ResearcherProfile.disease_areas, + ResearcherProfile.key_targets, + ResearcherProfile.keywords, + ResearcherProfile.private_profile_md, + ResearcherProfile.private_profile_seed, + ResearcherProfile.profile_version, + ).where(ResearcherProfile.user_id == uid) + ) + ).mappings().first() + + +async def _job_count(db, uid) -> int: + return ( + await db.execute(select(func.count()).select_from(Job).where(Job.user_id == uid)) + ).scalar_one() + + +async def _prefs(db, uid) -> dict: + rows = ( + await db.execute( + select( + EmailNotificationPreference.category, + EmailNotificationPreference.enabled, + EmailNotificationPreference.frequency, + ).where(EmailNotificationPreference.user_id == uid) + ) + ).all() + return {c: (e, f) for c, e, f in rows} + + +async def _snapshot(db, uid): + """Everything the 13 session-authenticated endpoints between them can change. + + One tuple, so a single equality covers "this endpoint touched the victim in + any way at all" without the sweep needing per-endpoint knowledge. + """ + user = await _user_row(db, uid) + if user is None: + return None + prof = await _prof(db, uid) + prof_t = None + if prof is not None: + prof_t = tuple( + tuple(v) if isinstance(v, list) else v for v in prof.values() + ) + return (tuple(user.values()), prof_t, await _job_count(db, uid), tuple(sorted( + (await _prefs(db, uid)).items() + ))) + + +# --- rendered-settings readers --------------------------------------------- + + +def _toggle(html: str, key: str) -> str: + m = re.search(rf'name="{key}_on" id="{key}_on" value="(\d)"', html) + assert m, f"no {key} toggle rendered on the settings page" + return m.group(1) + + +def _frequency(html: str, key: str) -> str: + parts = html.split(f'name="{key}_frequency"', 1) + assert len(parts) == 2, f"no {key} frequency select rendered on the settings page" + m = re.search(r'value="([a-z_]+)" selected', parts[1].split("", 1)[0]) + assert m, f"no option selected for {key}_frequency" + return m.group(1) + + +ALL_OFF = { + "proposal_review_on": "0", + "status_overview_on": "0", + "new_proposal_on": "0", + "news_updates_on": "0", +} + + +def _all_on(review="daily", overview="monthly") -> dict: + return { + "proposal_review_on": "1", + "proposal_review_frequency": review, + "status_overview_on": "1", + "status_overview_frequency": overview, + "new_proposal_on": "1", + "news_updates_on": "1", + } + + +# --------------------------------------------------------------------------- +# the endpoint inventory — the list the authorization sweeps iterate +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Ep: + method: str + path: str + build_data: Callable | None = None + auth: str = "session" # "session" | "token" (unsubscribe links carry no session) + onboarding_complete: bool = True # state the actor needs for this route to render + + @property + def label(self) -> str: + return f"{self.method} {self.path}" + + +def _onboarding_form(u): + return {"email": u.email or "", "research_summary": f"SWEEP-{u.orcid}"} + + +def _profile_form(u): + return { + "name": f"renamed-{u.orcid}", + "email": u.email or "", + "research_summary": f"SWEEP-{u.orcid}", + } + + +ENDPOINTS: list[Ep] = [ + # --- src/routers/onboarding.py (5) --- + Ep("GET", "/onboarding", onboarding_complete=False), + Ep("POST", "/onboarding/save-profile", _onboarding_form, onboarding_complete=False), + Ep("GET", "/onboarding/private-profile", onboarding_complete=False), + Ep( + "POST", + "/onboarding/private-profile", + lambda u: {"content": f"SWEEP-PRIVATE-{u.orcid}"}, + onboarding_complete=False, + ), + Ep("POST", "/onboarding/retry", lambda u: {}), + # --- src/routers/profile.py (6) --- + Ep("GET", "/profile"), + Ep("GET", "/profile/edit"), + Ep("POST", "/profile/save", _profile_form), + Ep("POST", "/profile/refresh", lambda u: {}), + Ep("GET", "/profile/delete-account"), + Ep("POST", "/profile/delete-account", lambda u: {"confirm": "delete"}), + # --- src/routers/settings.py (4) --- + Ep("GET", "/settings"), + Ep("POST", "/settings/save", lambda u: _all_on()), + Ep("GET", "/settings/unsubscribe/{token}", auth="token"), + Ep("POST", "/settings/unsubscribe/{token}", auth="token"), +] + +_ID = {e: e.label for e in ENDPOINTS} + + +async def _send(client, ep: Ep, actor, headers: dict, token: str | None = None): + """Fire ``ep`` as ``actor`` would. Empty headers means genuinely logged out.""" + if "Cookie" not in headers: + # httpx keeps a cookie jar; a Set-Cookie from an earlier authenticated + # request in the same test would otherwise silently log this one in. + client.cookies.clear() + path = ep.path.replace("{token}", token or "no-token") + if ep.method == "GET": + return await client.get(path, headers=headers) + data = ep.build_data(actor) if ep.build_data else None + return await client.post(path, data=data, headers=headers) + + +def test_the_endpoint_inventory_is_the_whole_first_run_surface(): + """The sweeps below are only as complete as this list. + + Read the routes off the three routers rather than trusting a hand-count, so + a 16th endpoint fails here loudly instead of quietly escaping the + authorization sweeps. + """ + live = set() + for prefix, module in ( + ("/onboarding", onboarding_router), + ("/profile", profile_router), + ("/settings", settings_router), + ): + for route in module.router.routes: + for method in route.methods: + if method in ("GET", "POST"): + live.add((method, prefix + route.path)) + + declared = {(e.method, e.path) for e in ENDPOINTS} + assert declared == live, ( + "the endpoint inventory has drifted from the routers; " + f"missing from the tests: {sorted(live - declared)}; " + f"no longer in the code: {sorted(declared - live)}" + ) + assert len(ENDPOINTS) == 15 + + # The two exemptions below are asserted, not assumed: unsubscribe links are + # clicked from an email client with no session. + assert {e.label for e in ENDPOINTS if e.auth == "token"} == { + "GET /settings/unsubscribe/{token}", + "POST /settings/unsubscribe/{token}", + } + + +# --------------------------------------------------------------------------- +# 1. the onboarding sequence +# --------------------------------------------------------------------------- + + +@pytest.fixture +async def newcomer(db_session): + return await factories.make_user( + db_session, + name="Newcomer Nadia", + email="nadia@example.org", + onboarding_complete=False, + access_status="allowed", + ) + + +async def test_the_onboarding_walk_completes_only_at_the_final_step( + client, db_session, newcomer, welcome_emails +): + """start -> ORCID-derived profile review -> private profile -> complete. + + onboarding_complete is checked after *every* step, so a router that set it + early (which would drop a user into /profile with a blank agent) fails here. + """ + h = _auth(newcomer.id) + + # Step 1 — the start page. The user arrived straight from the ORCID login + # with nothing generated yet; the page self-heals by enqueueing the + # generate_profile job the worker will pick up. + r = await client.get("/onboarding", headers=h) + assert r.status_code == 200 + assert "Building Your Profile" in r.text + assert await _job_count(db_session, newcomer.id) == 1 + assert await _flag(db_session, newcomer.id) is False + + # Step 2 — the worker's leg (ORCID + Anthropic) is out of scope here, so + # stand in for its result and re-request the page. + job = ( + await db_session.execute(select(Job).where(Job.user_id == newcomer.id)) + ).scalar_one() + job.status = "completed" + await factories.make_profile( + db_session, + user=newcomer, + research_summary="Generated summary about kinase signalling.", + techniques=["cryo-EM"], + keywords=["kinase"], + private_profile_md=None, + private_profile_seed="# Seeded private profile\n- prefers structural work", + ) + await db_session.flush() + + r = await client.get("/onboarding", headers=h) + assert r.status_code == 200 + assert "Generated summary about kinase signalling." in r.text + assert await _job_count(db_session, newcomer.id) == 1, "self-heal re-fired with a job present" + assert await _flag(db_session, newcomer.id) is False + + # Step 3 — the PI edits and saves the public profile. + r = await client.post( + "/onboarding/save-profile", + headers=h, + data={ + "email": "nadia@example.org", + "research_summary": "Edited by the PI during onboarding.", + "techniques": "cryo-EM, mass spec", + "experimental_models": "mouse", + "disease_areas": "cancer", + "key_targets": "KRAS", + "keywords": "kinase, structure", + }, + ) + assert r.status_code == 302 + assert r.headers["location"] == "/onboarding/private-profile" + prof = await _prof(db_session, newcomer.id) + assert prof["research_summary"] == "Edited by the PI during onboarding." + assert prof["techniques"] == ["cryo-EM", "mass spec"] + assert prof["keywords"] == ["kinase", "structure"] + assert prof["profile_version"] == 2 + assert await _flag(db_session, newcomer.id) is False, "saving the profile completed onboarding" + assert welcome_emails == [] + + # Step 4 — the private-profile editor offers the seed for review. + r = await client.get("/onboarding/private-profile", headers=h) + assert r.status_code == 200 + assert "Seeded private profile" in r.text + assert await _flag(db_session, newcomer.id) is False + + # Step 5 — saving the private profile is the step that finishes onboarding. + r = await client.post( + "/onboarding/private-profile", + headers=h, + data={"content": "# Nadia Lab — Private\n- no industry collaborations"}, + ) + assert r.status_code == 302 + assert r.headers["location"] == "/profile?onboarding_complete=1" + assert await _flag(db_session, newcomer.id) is True + prof = await _prof(db_session, newcomer.id) + assert prof["private_profile_md"] == "# Nadia Lab — Private\n- no industry collaborations" + assert prof["private_profile_seed"] is None, "the seed must be cleared once the PI edits it" + assert [e["to"] for e in welcome_emails] == ["nadia@example.org"] + + # And onboarding is now closed to this user. + for path in ("/onboarding", "/onboarding/private-profile"): + r = await client.get(path, headers=h) + assert r.status_code == 302 and r.headers["location"] == "/profile", path + + +@pytest.mark.parametrize( + "method,path,data", + [ + ("GET", "/onboarding", None), + ("GET", "/onboarding/private-profile", None), + ( + "POST", + "/onboarding/save-profile", + {"email": "nadia@example.org", "research_summary": "partial"}, + ), + ("POST", "/onboarding/retry", {}), + ], + ids=lambda v: v if isinstance(v, str) else "", +) +async def test_skipping_to_a_step_does_not_complete_onboarding( + client, db_session, newcomer, method, path, data +): + """Control for the walk above: none of the non-terminal steps may finish it. + + The positive control is in the same test — the terminal step is fired at the + end and must flip the flag, so a User whose onboarding_complete simply could + not change would fail rather than pass. + """ + h = _auth(newcomer.id) + await factories.make_profile(db_session, user=newcomer, private_profile_seed="seed") + + if method == "GET": + r = await client.get(path, headers=h) + else: + r = await client.post(path, headers=h, data=data) + assert r.status_code in (200, 302) + assert await _flag(db_session, newcomer.id) is False, f"{method} {path} completed onboarding" + + r = await client.post( + "/onboarding/private-profile", headers=h, data={"content": "done"} + ) + assert r.status_code == 302 + assert await _flag(db_session, newcomer.id) is True, "the terminal step no longer completes it" + + +async def test_the_start_page_enqueues_a_job_only_when_there_is_nothing_to_show( + client, db_session +): + """The self-heal in onboarding_start, and the two conditions that gate it.""" + allowed = await factories.make_user( + db_session, onboarding_complete=False, access_status="allowed" + ) + pending = await factories.make_user( + db_session, onboarding_complete=False, access_status="pending" + ) + has_profile = await factories.make_user( + db_session, onboarding_complete=False, access_status="allowed" + ) + await factories.make_profile(db_session, user=has_profile) + await db_session.flush() + + # positive: a stranded allowed user gets exactly one job, and only one. + assert (await client.get("/onboarding", headers=_auth(allowed.id))).status_code == 200 + assert await _job_count(db_session, allowed.id) == 1 + assert (await client.get("/onboarding", headers=_auth(allowed.id))).status_code == 200 + assert await _job_count(db_session, allowed.id) == 1 + + # controls: the two guards the self-heal is written with. + assert (await client.get("/onboarding", headers=_auth(pending.id))).status_code == 200 + assert await _job_count(db_session, pending.id) == 0, "self-heal ignored access_status" + assert (await client.get("/onboarding", headers=_auth(has_profile.id))).status_code == 200 + assert await _job_count(db_session, has_profile.id) == 0, "self-heal ignored the profile" + + +async def test_onboarding_save_profile_requires_a_valid_unused_email(client, db_session): + """Email is mandatory at onboarding and a rejected submission persists nothing.""" + other = await factories.make_user(db_session, email="taken@example.org") + u = await factories.make_user(db_session, email=None, onboarding_complete=False) + await db_session.flush() + h = _auth(u.id) + + for value, expected in ( + ("", "error=email_required"), + (" ", "error=email_required"), + ("not-an-email", "error=invalid_email"), + ("taken@example.org", "error=email_taken"), + ): + r = await client.post( + "/onboarding/save-profile", + headers=h, + data={"email": value, "research_summary": "should not be stored"}, + ) + assert r.status_code == 302 + assert expected in r.headers["location"], f"{value!r} -> {r.headers['location']}" + assert await _prof(db_session, u.id) is None, f"{value!r} persisted a profile anyway" + assert (await _user_row(db_session, u.id))["email"] is None + + # The email_taken branch is a cross-user write attempt: the other account + # must be untouched. + assert (await _user_row(db_session, other.id))["email"] == "taken@example.org" + + # positive control: a valid, unused address is accepted and stored. + r = await client.post( + "/onboarding/save-profile", + headers=h, + data={"email": "Fresh@Example.ORG", "research_summary": "stored"}, + ) + assert r.headers["location"] == "/onboarding/private-profile" + assert (await _user_row(db_session, u.id))["email"] == "fresh@example.org" + assert (await _prof(db_session, u.id))["research_summary"] == "stored" + + +async def test_the_private_profile_editor_falls_back_live_then_seed_then_disk_then_template( + client, db_session, export_dirs +): + """All four content sources in onboarding.private_profile, each against the next.""" + # 1. live markdown wins over the seed + live = await factories.make_user(db_session, onboarding_complete=False) + await factories.make_profile( + db_session, user=live, private_profile_md="LIVE-MD", private_profile_seed="SEED-MD" + ) + # 2. the seed is shown when there is no live markdown yet + seeded = await factories.make_user(db_session, onboarding_complete=False) + await factories.make_profile( + db_session, user=seeded, private_profile_md=None, private_profile_seed="SEED-ONLY" + ) + # 3. an on-disk profile from a pre-claim pilot lab + disk = await factories.make_user(db_session, onboarding_complete=False) + await factories.make_agent(db_session, user=disk, agent_id="diskpi", bot_name="DiskPiBot") + await factories.make_profile( + db_session, user=disk, private_profile_md=None, private_profile_seed=None + ) + export_dirs.private.mkdir(parents=True, exist_ok=True) + (export_dirs.private / "diskpi.md").write_text("ON-DISK-MD", encoding="utf-8") + # 4. nothing anywhere — the standard section template + blank = await factories.make_user( + db_session, name="Blank Slate", onboarding_complete=False + ) + await db_session.flush() + + r = await client.get("/onboarding/private-profile", headers=_auth(live.id)) + assert "LIVE-MD" in r.text and "SEED-MD" not in r.text + + r = await client.get("/onboarding/private-profile", headers=_auth(seeded.id)) + assert "SEED-ONLY" in r.text + + r = await client.get("/onboarding/private-profile", headers=_auth(disk.id)) + assert "ON-DISK-MD" in r.text + + r = await client.get("/onboarding/private-profile", headers=_auth(blank.id)) + assert "Blank Slate Lab — Private Profile" in r.text + assert "PI Behavioral Instructions" in r.text + # control: the template is not shown to someone who has real content. + r = await client.get("/onboarding/private-profile", headers=_auth(live.id)) + assert "PI Behavioral Instructions" not in r.text + + +async def test_the_terminal_step_flips_the_flag_and_welcomes_exactly_once( + client, db_session, newcomer, welcome_emails +): + """The replay control on ``_maybe_send_welcome``'s ``was_complete`` guard. + + Aimed at POST /onboarding/private-profile because that is the only terminal + step left: the duplicate POST /onboarding/complete this control used to fire + has been deleted. Nothing stops a replay of this one — unlike the GET, the + POST has no ``if current_user.onboarding_complete`` short-circuit — so the + guard is load-bearing and a second welcome email is reachable without it. + """ + h = _auth(newcomer.id) + r = await client.post("/onboarding/private-profile", headers=h, data={"content": "# Mine"}) + assert r.status_code == 302 + assert r.headers["location"] == "/profile?onboarding_complete=1" + assert await _flag(db_session, newcomer.id) is True + assert [e["to"] for e in welcome_emails] == ["nadia@example.org"] + + # control on the was_complete guard: a replay must not send a second welcome. + r = await client.post("/onboarding/private-profile", headers=h, data={"content": "# Mine"}) + assert r.status_code == 302 + assert len(welcome_emails) == 1, "the welcome email is sent again on every replay" + + +async def test_the_terminal_step_resumes_a_pending_invite_before_the_default_redirect( + client, db_session, newcomer +): + """The invite branch in save_private_profile. Control: no token -> /profile. + + Also inherited from the deleted POST /onboarding/complete, which carried the + same branch verbatim. + """ + h = _auth(newcomer.id) + r = await client.post("/onboarding/private-profile", headers=h, data={"content": "# Mine"}) + assert r.headers["location"] == "/profile?onboarding_complete=1" + + signer = TimestampSigner(get_settings().secret_key) + payload = {"user_id": str(newcomer.id), "pending_invite_token": "tok-123"} + cookie = signer.sign(base64.b64encode(json.dumps(payload).encode())).decode() + r = await client.post( + "/onboarding/private-profile", + headers={"Cookie": f"copi-session={cookie}"}, + data={"content": "# Mine"}, + ) + assert r.headers["location"] == "/invite/tok-123" + + +def _session_cookie(user_id, **extra) -> dict: + signer = TimestampSigner(get_settings().secret_key) + payload = {"user_id": str(user_id), **extra} + cookie = signer.sign(base64.b64encode(json.dumps(payload).encode())).decode() + return {"Cookie": f"copi-session={cookie}"} + + +async def test_finishing_onboarding_resumes_only_a_safe_post_login_destination( + client, db_session +): + """The terminal step honours post_login_redirect. It is attacker-influenced + (it comes off the /login query string), so the open-redirect guard has to + hold here too, not only in auth.py. + + This was parametrised over two endpoints until POST /onboarding/complete — + which duplicated the same resume block — was deleted. + """ + endpoint, data = "/onboarding/private-profile", {"content": "finished"} + for stashed, expected in ( + ("/settings", "/settings"), # positive: a real GET page resumes + ("https://evil.example.com/steal", "/profile?onboarding_complete=1"), + ("//evil.example.com/steal", "/profile?onboarding_complete=1"), + ("/logout", "/profile?onboarding_complete=1"), # deny-listed + ("/not-a-page", "/profile?onboarding_complete=1"), # not a GET route + ): + u = await factories.make_user(db_session, onboarding_complete=False) + await db_session.flush() + r = await client.post( + endpoint, + headers=_session_cookie(u.id, post_login_redirect=stashed), + data=data, + ) + assert r.status_code == 302 + assert r.headers["location"] == expected, f"{endpoint} with next={stashed!r}" + assert await _flag(db_session, u.id) is True + + +async def test_retry_enqueues_another_generate_profile_job(client, db_session, newcomer): + await factories.make_profile(db_session, user=newcomer) + await db_session.flush() + assert await _job_count(db_session, newcomer.id) == 0 # profile present, no self-heal + + r = await client.post("/onboarding/retry", headers=_auth(newcomer.id)) + assert r.status_code == 302 and r.headers["location"] == "/onboarding" + assert await _job_count(db_session, newcomer.id) == 1 + + job = ( + await db_session.execute(select(Job).where(Job.user_id == newcomer.id)) + ).scalar_one() + assert job.type == "generate_profile" + assert job.status == "pending" + assert job.payload["orcid"] == newcomer.orcid + + +# --------------------------------------------------------------------------- +# 2. src/routers/profile.py +# --------------------------------------------------------------------------- + + +async def test_profile_view_is_gated_on_onboarding(client, db_session): + u = await factories.make_user(db_session, onboarding_complete=False) + await factories.make_profile(db_session, user=u, research_summary="VIEW-SUMMARY") + await db_session.flush() + + r = await client.get("/profile", headers=_auth(u.id)) + assert r.status_code == 302 and r.headers["location"] == "/onboarding" + + # control: the identical request renders once onboarding is complete. + u.onboarding_complete = True + await db_session.flush() + r = await client.get("/profile", headers=_auth(u.id)) + assert r.status_code == 200 + assert "VIEW-SUMMARY" in r.text + + +async def test_profile_view_lists_publications_newest_first(client, db_session): + u = await factories.make_user(db_session, name="Pub Owner") + await factories.make_profile(db_session, user=u) + for year, title in ((2011, "Older paper"), (2021, "Newer paper")): + db_session.add( + Publication(user_id=u.id, title=title, journal="Cell", year=year) + ) + await db_session.flush() + + r = await client.get("/profile", headers=_auth(u.id)) + assert r.status_code == 200 + assert "Newer paper" in r.text and "Older paper" in r.text + assert r.text.index("Newer paper") < r.text.index("Older paper") + + +async def test_profile_edit_page_shows_the_current_values(client, db_session): + u = await factories.make_user(db_session, name="Edit Me", email="edit@example.org") + await factories.make_profile( + db_session, user=u, research_summary="EDITABLE-SUMMARY", techniques=["ct-a", "ct-b"] + ) + await db_session.flush() + + r = await client.get("/profile/edit", headers=_auth(u.id)) + assert r.status_code == 200 + assert "EDITABLE-SUMMARY" in r.text + assert "ct-a" in r.text and "ct-b" in r.text + assert "edit@example.org" in r.text + + +async def test_profile_save_persists_user_and_profile_fields_and_bumps_the_version( + client, db_session +): + u = await factories.make_user(db_session, name="Before Name", email="before@example.org") + await factories.make_profile(db_session, user=u, profile_version=3) + await db_session.flush() + + r = await client.post( + "/profile/save", + headers=_auth(u.id), + data={ + "name": "After Name", + "email": "after@example.org", + "institution": "New Institute", + "department": "New Dept", + "research_summary": "new summary", + "techniques": "t1, t2", + "experimental_models": "m1", + "disease_areas": "d1, d2", + "key_targets": "k1", + "keywords": "kw1, kw2", + }, + ) + assert r.status_code == 302 and r.headers["location"] == "/profile?saved=1" + + user = await _user_row(db_session, u.id) + assert user["name"] == "After Name" + assert user["email"] == "after@example.org" + assert user["institution"] == "New Institute" + assert user["department"] == "New Dept" + prof = await _prof(db_session, u.id) + assert prof["research_summary"] == "new summary" + assert prof["techniques"] == ["t1", "t2"] + assert prof["experimental_models"] == ["m1"] + assert prof["disease_areas"] == ["d1", "d2"] + assert prof["key_targets"] == ["k1"] + assert prof["keywords"] == ["kw1", "kw2"] + assert prof["profile_version"] == 4 + + +async def test_profile_save_rejects_a_bad_or_taken_email_and_persists_nothing( + client, db_session +): + other = await factories.make_user(db_session, email="owned@example.org", name="Owner") + u = await factories.make_user(db_session, name="Keep Me", email="keep@example.org") + await factories.make_profile(db_session, user=u, research_summary="untouched") + await db_session.flush() + h = _auth(u.id) + + for value, expected in ( + ("bogus", "error=invalid_email"), + ("owned@example.org", "error=email_taken"), + ): + r = await client.post( + "/profile/save", + headers=h, + data={"name": "Hijacked", "email": value, "research_summary": "hijacked"}, + ) + assert r.status_code == 302 + assert expected in r.headers["location"] + user = await _user_row(db_session, u.id) + assert (user["name"], user["email"]) == ("Keep Me", "keep@example.org") + assert (await _prof(db_session, u.id))["research_summary"] == "untouched" + + # the taken-email attempt must not have moved the other account's address + assert (await _user_row(db_session, other.id))["email"] == "owned@example.org" + + # control: a legitimate save goes through, so "nothing persisted" above is + # not just a route that never writes. + r = await client.post( + "/profile/save", + headers=h, + data={"name": "Renamed", "email": "keep@example.org", "research_summary": "written"}, + ) + assert r.headers["location"] == "/profile?saved=1" + assert (await _user_row(db_session, u.id))["name"] == "Renamed" + + +async def test_profile_refresh_enqueues_exactly_one_job(client, db_session): + u = await factories.make_user(db_session) + await db_session.flush() + r = await client.post("/profile/refresh", headers=_auth(u.id)) + assert r.status_code == 302 and r.headers["location"] == "/profile?refreshing=1" + assert await _job_count(db_session, u.id) == 1 + + +async def test_delete_account_confirmation_page_renders(client, db_session): + u = await factories.make_user(db_session) + await db_session.flush() + r = await client.get("/profile/delete-account", headers=_auth(u.id)) + assert r.status_code == 200 + assert "Delete Account" in r.text + + +async def test_delete_account_needs_the_confirmation_word(client, db_session): + u = await factories.make_user(db_session) + await factories.make_profile(db_session, user=u) + db_session.add(Publication(user_id=u.id, title="doomed paper")) + await db_session.flush() + h = _auth(u.id) + + for word in ("", "yes", "DELETE ME"): + r = await client.post("/profile/delete-account", headers=h, data={"confirm": word}) + assert r.status_code == 302 + assert "error=1" in r.headers["location"], word + assert await _user_row(db_session, u.id) is not None, f"{word!r} deleted the account" + + # control: the word does delete, cascading to the profile and publications. + # (Case-insensitive by design — profile.py lowercases the input.) + r = await client.post("/profile/delete-account", headers=h, data={"confirm": "Delete"}) + assert r.status_code == 302 and r.headers["location"] == "/login?deleted=1" + assert await _user_row(db_session, u.id) is None + assert await _prof(db_session, u.id) is None + assert ( + await db_session.execute( + select(func.count()).select_from(Publication).where(Publication.user_id == u.id) + ) + ).scalar_one() == 0 + + +# --------------------------------------------------------------------------- +# 3. src/services/profile_export.py — no test referenced this module at all +# --------------------------------------------------------------------------- + + +async def test_the_public_export_round_trips_and_never_carries_the_private_profile( + db_session, export_dirs +): + """The highest-consequence assertion in this task. + + The public export is what any agent (and anything downstream of the agent) + reads. The private profile is the PI's behavioural instructions and must not + appear in it. The control is the private export writing the same canary — + without it, an export that produced an empty file would satisfy "no leak". + """ + user = await factories.make_user( + db_session, name="Export Pi", institution="Scripps", department="Mol Bio" + ) + prof = await factories.make_profile( + db_session, + user=user, + research_summary="Summary of the lab's work.", + techniques=["cryo-EM", "MD simulation"], + experimental_models=["zebrafish"], + disease_areas=["glioma"], + key_targets=["EGFR"], + keywords=["kinase", "structure"], + grant_titles=["R01 Something Important"], + private_profile_md="PRIVATE-CANARY-never-export-me", + ) + pubs = [ + Publication( + user_id=user.id, + title="A structural paper.", + journal="Cell", + year=2020, + doi="10.1016/j.cell.2020.01.001", + ) + ] + + path = profile_export.export_profile_to_markdown(user, prof, "exportpi", publications=pubs) + assert path == export_dirs.public / "exportpi.md" + text = path.read_text(encoding="utf-8") + + for expected in ( + "Export Pi Lab — Public Profile", + "**PI:** Export Pi", + "**Institution:** Scripps", + "**Department:** Mol Bio", + "Summary of the lab's work.", + "- cryo-EM", + "- MD simulation", + "- zebrafish", + "- glioma", + "- EGFR", + "kinase, structure", + "- R01 Something Important", + "A structural paper. *Cell*. (2020). https://doi.org/10.1016/j.cell.2020.01.001", + ): + assert expected in text, f"the public export dropped {expected!r}" + + assert "PRIVATE-CANARY-never-export-me" not in text, ( + "the public profile export leaks private_profile_md" + ) + + # CONTROL — the private export does contain it, so the assertion above is + # about where the content goes, not about the export producing nothing. + ppath = profile_export.export_private_profile(user, prof, "exportpi") + assert ppath == export_dirs.private / "exportpi.md" + assert "PRIVATE-CANARY-never-export-me" in ppath.read_text(encoding="utf-8") + + +async def test_both_exports_are_gated_on_an_agent_registry_id(db_session, export_dirs): + user = await factories.make_user(db_session) + prof = await factories.make_profile(db_session, user=user, private_profile_md="x") + + assert profile_export.export_profile_to_markdown(user, prof, None) is None + assert profile_export.export_private_profile(user, prof, None) is None + assert not export_dirs.public.exists() and not export_dirs.private.exists() + + # control: with an agent id both write. + assert profile_export.export_profile_to_markdown(user, prof, "gated") is not None + assert profile_export.export_private_profile(user, prof, "gated") is not None + + +async def test_the_private_export_skips_an_empty_private_profile(db_session, export_dirs): + user = await factories.make_user(db_session) + prof = await factories.make_profile(db_session, user=user, private_profile_md=None) + assert profile_export.export_private_profile(user, prof, "emptypi") is None + assert not (export_dirs.private / "emptypi.md").exists() + + # control + prof.private_profile_md = "now there is content" + assert profile_export.export_private_profile(user, prof, "emptypi") is not None + assert (export_dirs.private / "emptypi.md").read_text(encoding="utf-8").startswith( + "now there is content" + ) + + +async def test_the_export_drops_a_doi_that_contradicts_the_journal(db_session): + """_validate_doi_journal, through the export. A DOI attributed to the wrong + journal is a paper attributed to the wrong lab.""" + user = await factories.make_user(db_session) + prof = await factories.make_profile(db_session, user=user) + + mismatch = Publication( + user_id=user.id, + title="Mislinked paper", + journal="Cell", + year=2019, + doi="10.1126/science.aaa1234", + pmid="31111111", + ) + text = profile_export.export_profile_to_markdown( + user, prof, "doipi", publications=[mismatch] + ).read_text(encoding="utf-8") + assert "https://pubmed.ncbi.nlm.nih.gov/31111111/" in text + assert "10.1126/science.aaa1234" not in text + + # control: the same DOI on the journal it belongs to is kept. + match = Publication( + user_id=user.id, + title="Correctly linked paper", + journal="Science", + year=2019, + doi="10.1126/science.aaa1234", + pmid="31111111", + ) + text = profile_export.export_profile_to_markdown( + user, prof, "doipi", publications=[match] + ).read_text(encoding="utf-8") + assert "https://doi.org/10.1126/science.aaa1234" in text + + +async def test_the_export_keeps_the_twenty_most_recent_publications(db_session): + user = await factories.make_user(db_session) + prof = await factories.make_profile(db_session, user=user) + pubs = [ + Publication(user_id=user.id, title=f"Paper {year}", journal="J", year=year) + for year in range(1990, 2015) # 25 of them + ] + text = profile_export.export_profile_to_markdown( + user, prof, "manypi", publications=pubs + ).read_text(encoding="utf-8") + + assert "Paper 2014" in text # newest kept + assert "Paper 1990" not in text # oldest dropped + assert text.count("- Paper ") == 20 + + +async def test_saving_the_profile_writes_the_export_and_records_a_public_revision( + client, db_session, export_dirs +): + """The route side of the export, plus its AgentRegistry gate.""" + user = await factories.make_user(db_session, name="Route Pi") + agent = await factories.make_agent( + db_session, user=user, agent_id="routepi", bot_name="RoutePiBot" + ) + await factories.make_profile(db_session, user=user) + await db_session.flush() + + r = await client.post( + "/profile/save", + headers=_auth(user.id), + data={ + "name": "Route Pi", + "email": user.email, + "research_summary": "EXPORTED-VIA-ROUTE", + "techniques": "route-technique", + }, + ) + assert r.status_code == 302 + + written = (export_dirs.public / "routepi.md").read_text(encoding="utf-8") + assert "EXPORTED-VIA-ROUTE" in written + assert "- route-technique" in written + + revs = ( + await db_session.execute( + select(ProfileRevision).where(ProfileRevision.agent_registry_id == agent.id) + ) + ).scalars().all() + assert [rv.profile_type for rv in revs] == ["public"] + assert revs[0].mechanism == "web" + assert revs[0].changed_by_user_id == user.id + assert revs[0].content == written, "the revision must record what was exported" + + # control: the same save by a user with no AgentRegistry writes no file and + # no revision, which is why the gate exists. + plain = await factories.make_user(db_session, name="No Agent") + await factories.make_profile(db_session, user=plain) + await db_session.flush() + r = await client.post( + "/profile/save", + headers=_auth(plain.id), + data={"name": "No Agent", "email": plain.email, "research_summary": "no export"}, + ) + assert r.status_code == 302 + assert sorted(p.name for p in export_dirs.public.iterdir()) == ["routepi.md"] + assert ( + await db_session.execute(select(func.count()).select_from(ProfileRevision)) + ).scalar_one() == 1 + + +async def test_saving_the_private_profile_writes_the_private_file_and_a_private_revision( + client, db_session, export_dirs +): + user = await factories.make_user(db_session, onboarding_complete=False) + agent = await factories.make_agent( + db_session, user=user, agent_id="privpi", bot_name="PrivPiBot" + ) + await factories.make_profile(db_session, user=user, private_profile_md=None) + await db_session.flush() + + r = await client.post( + "/onboarding/private-profile", + headers=_auth(user.id), + data={"content": "PRIVATE-VIA-ROUTE"}, + ) + assert r.status_code == 302 + assert (export_dirs.private / "privpi.md").read_text(encoding="utf-8") == ( + "PRIVATE-VIA-ROUTE\n" + ) + revs = ( + await db_session.execute( + select(ProfileRevision).where(ProfileRevision.agent_registry_id == agent.id) + ) + ).scalars().all() + assert [rv.profile_type for rv in revs] == ["private"] + assert revs[0].content == "PRIVATE-VIA-ROUTE" + + # control: clearing the private profile writes no file and records no + # revision, but still completes onboarding. + other = await factories.make_user(db_session, onboarding_complete=False) + await factories.make_agent( + db_session, user=other, agent_id="emptyroutepi", bot_name="EmptyRoutePiBot" + ) + await db_session.flush() + r = await client.post( + "/onboarding/private-profile", headers=_auth(other.id), data={"content": " "} + ) + assert r.status_code == 302 + assert not (export_dirs.private / "emptyroutepi.md").exists() + assert ( + await db_session.execute(select(func.count()).select_from(ProfileRevision)) + ).scalar_one() == 1 + assert await _flag(db_session, other.id) is True + + +# --------------------------------------------------------------------------- +# 4. src/routers/settings.py +# --------------------------------------------------------------------------- + + +async def test_every_setting_persists_and_is_reflected_on_the_next_request( + client, db_session +): + u = await factories.make_user(db_session) + await db_session.flush() + h = _auth(u.id) + + # The GET before any POST shows CATEGORY_DEFAULTS. + r = await client.get("/settings", headers=h) + assert r.status_code == 200 + assert _toggle(r.text, "proposal_review") == "1" + assert _frequency(r.text, "proposal_review") == "weekly" + assert _toggle(r.text, "status_overview") == "1" + assert _toggle(r.text, "new_proposal") == "0" + assert _toggle(r.text, "news_updates") == "1" + + # Turn everything on, with non-default frequencies. + r = await client.post("/settings/save", headers=h, data=_all_on("daily", "monthly")) + assert r.status_code == 302 and r.headers["location"] == "/settings?saved=1" + assert (await _user_row(db_session, u.id))["email_notification_frequency"] == "daily" + prefs = await _prefs(db_session, u.id) + assert prefs["status_overview"] == (True, "monthly") + assert prefs["new_proposal"][0] is True + assert prefs["news_updates"][0] is True + + r = await client.get("/settings", headers=h) + assert _frequency(r.text, "proposal_review") == "daily" + assert _frequency(r.text, "status_overview") == "monthly" + for key in ("proposal_review", "status_overview", "new_proposal", "news_updates"): + assert _toggle(r.text, key) == "1", key + + # Control for the above: turning everything off must also round-trip, so + # "reflected on the next request" is not satisfied by a page hard-coded on. + r = await client.post("/settings/save", headers=h, data=ALL_OFF) + assert r.status_code == 302 + assert (await _user_row(db_session, u.id))["email_notification_frequency"] == "off" + prefs = await _prefs(db_session, u.id) + assert prefs["status_overview"] == (False, "off") + assert prefs["new_proposal"][0] is False + assert prefs["news_updates"][0] is False + + r = await client.get("/settings", headers=h) + for key in ("proposal_review", "status_overview", "new_proposal", "news_updates"): + assert _toggle(r.text, key) == "0", key + + +async def test_the_settings_page_reads_defaults_without_inserting_rows(client, db_session): + """The GET is documented as insert-free; the POST is what materialises rows.""" + u = await factories.make_user(db_session) + await db_session.flush() + h = _auth(u.id) + + assert (await client.get("/settings", headers=h)).status_code == 200 + assert await _prefs(db_session, u.id) == {}, "the settings GET inserted preference rows" + + # control + assert (await client.post("/settings/save", headers=h, data=ALL_OFF)).status_code == 302 + assert set(await _prefs(db_session, u.id)) == { + "status_overview", + "new_proposal", + "news_updates", + } + + +async def test_an_invalid_review_frequency_falls_back_to_weekly(client, db_session): + u = await factories.make_user(db_session) + await db_session.flush() + h = _auth(u.id) + + await client.post( + "/settings/save", + headers=h, + data={**ALL_OFF, "proposal_review_on": "1", "proposal_review_frequency": "hourly"}, + ) + assert (await _user_row(db_session, u.id))["email_notification_frequency"] == "weekly" + + # control: a valid value is stored as given, not coerced. + await client.post( + "/settings/save", + headers=h, + data={**ALL_OFF, "proposal_review_on": "1", "proposal_review_frequency": "biweekly"}, + ) + assert (await _user_row(db_session, u.id))["email_notification_frequency"] == "biweekly" + + +async def test_re_enabling_review_emails_clears_a_system_pause(client, db_session): + u = await factories.make_user( + db_session, email_notification_frequency="off", + email_notifications_paused_by_system=True, + ) + await db_session.flush() + h = _auth(u.id) + + # control first: staying off leaves the pause in place. + await client.post("/settings/save", headers=h, data=ALL_OFF) + assert (await _user_row(db_session, u.id))["email_notifications_paused_by_system"] is True + + await client.post( + "/settings/save", + headers=h, + data={**ALL_OFF, "proposal_review_on": "1", "proposal_review_frequency": "weekly"}, + ) + assert (await _user_row(db_session, u.id))["email_notifications_paused_by_system"] is False + + +async def test_changing_the_review_frequency_resets_the_missed_counter(client, db_session): + u = await factories.make_user(db_session, email_notification_frequency="weekly") + db_session.add(EmailEngagementTracker(user_id=u.id, consecutive_missed=3)) + await db_session.flush() + h = _auth(u.id) + + async def missed(): + return ( + await db_session.execute( + select(EmailEngagementTracker.consecutive_missed).where( + EmailEngagementTracker.user_id == u.id + ) + ) + ).scalar_one() + + # control: re-saving the same frequency must not reset it. + await client.post( + "/settings/save", + headers=h, + data={**ALL_OFF, "proposal_review_on": "1", "proposal_review_frequency": "weekly"}, + ) + assert await missed() == 3 + + await client.post( + "/settings/save", + headers=h, + data={**ALL_OFF, "proposal_review_on": "1", "proposal_review_frequency": "daily"}, + ) + assert await missed() == 0 + + +async def test_the_unsubscribe_get_is_read_only_and_the_post_performs_it(client, db_session): + """Email-security scanners fetch every link; the GET must not mutate.""" + u = await factories.make_user(db_session, email_notification_frequency="weekly") + await db_session.flush() + token = _generate_unsubscribe_token(str(u.id)) + + r = await client.get(f"/settings/unsubscribe/{token}") + assert r.status_code == 200 + assert "Invalid or expired" not in r.text + assert (await _user_row(db_session, u.id))["email_notification_frequency"] == "weekly", ( + "the unsubscribe GET unsubscribed the user" + ) + + # control: the POST does what the GET refused to. + r = await client.post(f"/settings/unsubscribe/{token}") + assert r.status_code == 200 + assert (await _user_row(db_session, u.id))["email_notification_frequency"] == "off" + + +async def test_an_unsubscribe_token_only_affects_the_user_it_was_minted_for( + client, db_session +): + a = await factories.make_user(db_session, email_notification_frequency="weekly") + b = await factories.make_user(db_session, email_notification_frequency="weekly") + await db_session.flush() + + r = await client.post(f"/settings/unsubscribe/{_generate_unsubscribe_token(str(a.id))}") + assert r.status_code == 200 + assert (await _user_row(db_session, a.id))["email_notification_frequency"] == "off" + assert (await _user_row(db_session, b.id))["email_notification_frequency"] == "weekly" + + # control: b's own token turns b off, so "b untouched" is not just an + # endpoint that never works. + await client.post(f"/settings/unsubscribe/{_generate_unsubscribe_token(str(b.id))}") + assert (await _user_row(db_session, b.id))["email_notification_frequency"] == "off" + + +async def test_unsubscribe_handles_a_token_for_a_user_that_no_longer_exists( + client, db_session +): + ghost = await factories.make_user(db_session) + token = _generate_unsubscribe_token(str(ghost.id)) + await db_session.delete(ghost) + await db_session.flush() + + assert "User not found" in (await client.get(f"/settings/unsubscribe/{token}")).text + r = await client.post(f"/settings/unsubscribe/{token}") + assert r.status_code == 404 and "User not found" in r.text + + +# --------------------------------------------------------------------------- +# 5. authorization, asserted per endpoint +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("ep", ENDPOINTS, ids=_ID.get) +async def test_every_endpoint_that_needs_a_session_redirects_a_logged_out_caller( + client, db_session, ep +): + """Half one of the sweep, per endpoint. + + The redirect alone is not enough: /profile/delete-account redirects to + /login on success too, so a route that let an anonymous caller through + would still land on a Location starting with "/login". Each case therefore + also asserts the anonymous request changed nothing, and pairs that with the + same request carrying a session, which must do the thing — otherwise an + endpoint that is simply broken would score as correctly protected. + """ + u = await factories.make_user( + db_session, + email=f"sweep-{ep.method.lower()}{abs(hash(ep.path)) % 10**8}@example.org", + onboarding_complete=ep.onboarding_complete, + access_status="allowed", + ) + await factories.make_profile(db_session, user=u) + await db_session.flush() + token = _generate_unsubscribe_token(str(u.id)) + + before = await _snapshot(db_session, u.id) + logged_out = await _send(client, ep, u, {}, token=token) + after_anonymous = await _snapshot(db_session, u.id) + + if ep.auth == "token": + # Documented exemption: unsubscribe links are clicked from an email + # client. Their protection is the signed token, tested in the next sweep. + assert logged_out.status_code == 200, ep.label + assert (await _send(client, ep, u, _auth(u.id), token=token)).status_code == 200 + return + + assert logged_out.status_code == 302, f"{ep.label} served a logged-out caller" + assert logged_out.headers["location"].startswith("/login"), ep.label + assert after_anonymous == before, f"{ep.label} acted on behalf of a logged-out caller" + + logged_in = await _send(client, ep, u, _auth(u.id), token=token) + after_session = await _snapshot(db_session, u.id) + if ep.method == "POST": + assert after_session != before, ( + f"{ep.label} does nothing even with a valid session, so 'nothing " + "happened for the anonymous caller' proves nothing" + ) + else: + assert logged_in.status_code == 200, ( + f"{ep.label} does not render even with a valid session, so the " + "redirect above is not evidence of authorization" + ) + + +@pytest.mark.parametrize("ep", ENDPOINTS, ids=_ID.get) +async def test_no_logged_in_user_can_read_or_write_another_users_data(client, db_session, ep): + """Half two, the one worth most, per endpoint. + + None of these routes takes a target user id, so the only handle a caller has + on another identity is the ``copi-impersonate`` cookie, which + get_current_user honours for admins only. Each case fires the attacker's + request with that cookie pointed at the victim and asserts the effect landed + on the attacker; then fires the identical request as a real admin and + asserts it DOES land on the victim — so a renamed or removed cookie could + not make the negative half pass vacuously. + + For the two unsubscribe endpoints, which carry no session at all, the + cross-user question is instead whether the attacker can mint a token for the + victim; three forgeries are tried and the genuine token is the control. + """ + victim = await factories.make_user( + db_session, + name="Victim Alpha", + email="victim@example.org", + institution="Victim Institute", + onboarding_complete=ep.onboarding_complete, + access_status="allowed", + ) + await factories.make_profile( + db_session, + user=victim, + research_summary="VICTIM-SECRET-SUMMARY", + private_profile_md="VICTIM-PRIVATE-SECRET", + private_profile_seed=None, + ) + # A completed job, so GET /onboarding renders the review form rather than + # self-healing a new job into the victim's snapshot. + db_session.add( + Job(type="generate_profile", status="completed", user_id=victim.id, payload={}) + ) + + attacker = await factories.make_user( + db_session, + name="Attacker Beta", + email="attacker@example.org", + is_admin=False, + onboarding_complete=ep.onboarding_complete, + access_status="allowed", + ) + await factories.make_profile( + db_session, + user=attacker, + research_summary="attacker summary", + private_profile_md="attacker private", + private_profile_seed=None, + ) + db_session.add( + Job(type="generate_profile", status="completed", user_id=attacker.id, payload={}) + ) + await db_session.flush() + + victim_before = await _snapshot(db_session, victim.id) + attacker_before = await _snapshot(db_session, attacker.id) + + if ep.auth == "token": + secret = get_settings().secret_key + forgeries = { + "the bare user id": str(victim.id), + "a token signed with another secret": URLSafeTimedSerializer( + "not-the-real-secret", salt="unsubscribe" + ).dumps(str(victim.id)), + "a token signed with the wrong salt": URLSafeTimedSerializer( + secret, salt="not-unsubscribe" + ).dumps(str(victim.id)), + } + for how, tok in forgeries.items(): + r = await _send(client, ep, victim, _auth(attacker.id), token=tok) + assert "Invalid or expired" in r.text, f"{ep.label} accepted {how}" + assert await _snapshot(db_session, victim.id) == victim_before, ( + f"{ep.label} let {how} change another user's settings" + ) + + # CONTROL — the genuine token is accepted, so the rejections above are + # about the signature and not about a route that rejects everything. + genuine = _generate_unsubscribe_token(str(victim.id)) + r = await _send(client, ep, victim, {}, token=genuine) + assert r.status_code == 200 and "Invalid or expired" not in r.text + after = await _snapshot(db_session, victim.id) + if ep.method == "GET": + assert after == victim_before, "the unsubscribe GET is supposed to be read-only" + else: + assert after != victim_before, "the genuine token did nothing" + return + + r = await _send(client, ep, attacker, _auth_as(attacker.id, victim.id)) + assert r.status_code in (200, 302), f"{ep.label} errored for the attacker: {r.status_code}" + + victim_after = await _snapshot(db_session, victim.id) + attacker_after = await _snapshot(db_session, attacker.id) + assert victim_after == victim_before, ( + f"AUTHORIZATION HOLE: {ep.label} let a non-admin change another user's data" + ) + + if ep.method == "GET": + assert r.status_code == 200, f"{ep.label} did not render for the attacker" + for leaked in ("Viewing as Victim Alpha", "VICTIM-SECRET-SUMMARY", + "VICTIM-PRIVATE-SECRET", "Victim Institute"): + assert leaked not in r.text, ( + f"AUTHORIZATION HOLE: {ep.label} showed a non-admin {leaked!r}" + ) + else: + assert attacker_after != attacker_before, ( + f"{ep.label} changed nothing for the caller either, so the " + "'victim unchanged' assertion above proves nothing" + ) + + # CONTROL — the same cookie, from a real admin, must reach the victim. + admin = await factories.make_user( + db_session, name="Real Admin", email="realadmin@example.org", is_admin=True + ) + await db_session.flush() + control_before = await _snapshot(db_session, victim.id) + r2 = await _send(client, ep, victim, _auth_as(admin.id, victim.id)) + control_after = await _snapshot(db_session, victim.id) + + if ep.method == "GET": + assert r2.status_code == 200, ep.label + assert "Viewing as Victim Alpha" in r2.text, ( + "the copi-impersonate cookie is inert even for an admin, so the " + "negative assertions above are not testing anything" + ) + else: + assert control_after != control_before, ( + "the copi-impersonate cookie is inert even for an admin, so the " + "negative assertions above are not testing anything" + ) diff --git a/tests/integration/test_pi_inbox.py b/tests/integration/test_pi_inbox.py new file mode 100644 index 0000000..deadedd --- /dev/null +++ b/tests/integration/test_pi_inbox.py @@ -0,0 +1,98 @@ +"""Integration tests for the DB-native PI inbox (src/services/pi_inbox.py). + +These helpers are how a PI's web-authored messages and DMs enter the simulation +when Slack is off — the engine ingests the rows they write. Exercised against the +real migrated Postgres so the actual agent_messages / pi_dm_messages schema +(including the 0019/0020 columns) is validated. See specs/local-db-conversations.md. +""" + +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import select + +from src.models import AgentMessage, PiDmMessage +from src.services.pi_inbox import ( + get_latest_run_id, + record_pi_dm, + record_pi_message, + web_pi_user_id, +) +from tests import factories + +pytestmark = pytest.mark.integration + + +async def test_get_latest_run_id_returns_most_recent(db_session): + # Explicit started_at: now() is the (shared) txn timestamp, so ordering + # between two same-transaction rows would otherwise be ambiguous. + now = datetime.now(UTC) + await factories.make_simulation_run(db_session, started_at=now - timedelta(minutes=5)) + r2 = await factories.make_simulation_run(db_session, started_at=now) + latest = await get_latest_run_id(db_session) + assert latest == r2.id + + +async def test_record_pi_message_resolves_channel_and_writes_human_row(db_session): + run = await factories.make_simulation_run(db_session) + # A known channel with collab_private visibility should be picked up. + await factories.make_agent_channel( + db_session, run=run, channel_name="general", channel_id="C-GEN", + visibility="collab_private", + ) + msg = await record_pi_message( + db_session, run_id=run.id, channel_name="general", + content="please prioritize the kinase panel", sender_name="Dr Smoke (PI)", + ) + await db_session.flush() + + assert msg.is_bot is False # human/PI message + assert msg.agent_id is None # NULL sender_agent_id + assert msg.channel_id == "C-GEN" # resolved from agent_channels + assert msg.visibility == "collab_private" + assert msg.phase == "new_post" # top-level (no thread_ts) + assert msg.posted_at > 0 and msg.message_ts + + row = (await db_session.execute( + select(AgentMessage).where(AgentMessage.message_ts == msg.message_ts) + )).scalar_one() + assert row.content == "please prioritize the kinase panel" + assert row.sender_name == "Dr Smoke (PI)" + + +async def test_record_pi_message_reply_and_local_channel_fallback(db_session): + run = await factories.make_simulation_run(db_session) + # No agent_channels row for this name -> local: id, public visibility. + msg = await record_pi_message( + db_session, run_id=run.id, channel_name="drug-repurposing", + content="following up here", sender_name="PI", thread_ts="123.456", + ) + assert msg.channel_id == "local:drug-repurposing" + assert msg.visibility == "public" + assert msg.thread_ts == "123.456" + assert msg.phase == "thread_reply" # has a thread_ts + + +async def test_record_pi_dm_inbound_and_outbound(db_session): + run = await factories.make_simulation_run(db_session) + uid = uuid.uuid4() + inbound = await record_pi_dm( + db_session, run_id=run.id, agent_id="su", pi_user_id=web_pi_user_id(uid), + direction="inbound", content="always cc me on proposals", sender_name="PI", + ) + await record_pi_dm( + db_session, run_id=run.id, agent_id="su", pi_user_id=web_pi_user_id(uid), + direction="outbound", content="noted — will do", sender_name="SuBot", + ) + await db_session.flush() + + assert inbound.pi_user_id == f"local:{uid}" + rows = (await db_session.execute( + select(PiDmMessage).where(PiDmMessage.simulation_run_id == run.id) + .order_by(PiDmMessage.posted_at.asc()) + )).scalars().all() + assert [r.direction for r in rows] == ["inbound", "outbound"] + assert rows[0].content == "always cc me on proposals" + assert rows[1].agent_id == "su" + assert all(r.ts and r.posted_at > 0 for r in rows) diff --git a/tests/integration/test_profile_pipeline_live.py b/tests/integration/test_profile_pipeline_live.py new file mode 100644 index 0000000..1ce2fde --- /dev/null +++ b/tests/integration/test_profile_pipeline_live.py @@ -0,0 +1,1221 @@ +"""Task T4 — the profile pipeline, end to end and live. + +`live_api` **and** `real_llm`. This is the first test in the system where ORCID, PubMed +and Anthropic run together against a real database. It is the actual production path for +onboarding a PI: `worker.execute_generate_profile` calls exactly this function. + +Everything else that touches `run_profile_pipeline` — the four golden masters in +`tests/characterization/test_profile_pipeline_gm.py` — replaces every external boundary +with a fake. Those tests prove the pipeline wires its own parts together. They cannot +prove that the parts still fit the world: the fakes return the shapes their author +believed ORCID, NCBI and Claude produce (Rule L1). T4.5 reconciles the two. + +**The assertion that matters most** is in T4.1: the generated summary must mention +something *from the fetched works*. Without it, a pipeline that ignored its entire input +and hallucinated a plausible profile from the researcher's name would pass every other +assertion in this file. The expected vocabulary is derived from a live ORCID fetch the +test performs itself, never hardcoded. + +That assertion needed two attempts, and the first one was wrong. Matching the corpus +vocabulary against a hand-written decoy only proves the matcher *can* say no. Measured on +2026-07-30: given nothing but "Lisa Racki, Scripps Research Institute, Integrative +Structural and Computational Biology" and an empty publication list, Claude Opus returns +a confident, `_validate_profile`-passing profile that already contains three of the seven +derived corpus terms (chromatin, histone, remodeling), from what it remembers about her. +So the real control is empirical and runs inside T4.1: the same model is asked for a +name-only profile on the same day, and the run must produce corpus vocabulary the +name-only run did NOT. Only that difference is attributable to the pipeline having read +its inputs. Measured the same day, the real run covered all 7 terms and the difference +was {polyphosphate, granule, pseudomonas, aeruginosa} — the researcher's independent +programme, which the model does not recall but the fetched abstracts supply. + +Rule L3: each assertion message names which of provider-down / rate-limited / +schema-changed / our-code-broken it observed. + +Cost: 8 real Anthropic calls for the whole file — 3 for T4.1 (public synthesis, private +seed, and the name-only control), 3 for T4.2, 0 for T4.3, 2 for T4.4, 0 for T4.5. T4.5 +makes two further Anthropic *requests* that spend no tokens: they are deliberately +unauthenticated, which is how the GM #2 failure path is reproduced against the real API +rather than against a fake that raises RuntimeError. + +Run it with: + + docker compose exec -T -e LIVE_API_TESTS=1 -e ANTHROPIC_API_KEY=sk-ant-... \\ + -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_b2 \\ + app python -m pytest tests/integration/test_profile_pipeline_live.py -q \\ + -m 'live_api and real_llm' +""" + +import hashlib +import os +import re +import uuid +from pathlib import Path +from urllib.parse import urlparse + +import anthropic +import httpx +import pytest +import respx +from sqlalchemy import select + +from src.models import ProfileRevision, Publication, ResearcherProfile +from src.services import orcid as orcid_service +from src.services import profile_pipeline, pubmed +from tests import factories + +pytestmark = [ + pytest.mark.integration, + pytest.mark.live_api, + pytest.mark.real_llm, + pytest.mark.skipif( + not os.environ.get("ANTHROPIC_API_KEY"), + reason="no ANTHROPIC_API_KEY — real-API tests are opt-in and cost money", + ), +] + + +# --------------------------------------------------------------------------- the record +# +# Lisa Racki — Scripps Research, Integrative Structural and Computational Biology. +# +# Why not Josiah Carberry (0000-0002-1825-0097), the record T1 uses: that persona has no +# works at all, so `pubs_for_synthesis` is empty and the synthesis path — the entire +# point of T4 — never runs. Carberry can only prove the pipeline survives an empty +# corpus, which is what T4.4 covers by other means. +# +# Why this record is a defensible choice (Rule L2 — permanence, and nothing pinned): +# +# * It is already a production dependency of this repository. It is listed in +# `orcids.txt` under "Pilot lab ORCIDs — Scripps Research" (verified 2026-03-21), so +# if it were ever deleted or made private the product would break long before this +# test noticed, and the failure would be the *correct* signal rather than noise. +# * It is small and slow-growing — 12 work entries spanning 2002-2025 as of +# 2026-07-30, roughly one paper a year. Small matters twice over: it bounds the token +# spend, and it bounds the number of unthrottled NCBI requests `run_profile_pipeline` +# fires (see `_ncbi_get`, which paces itself at ~8 req/s against a 3 req/s anonymous +# policy limit — a large corpus would be the thing that gets this IP blocked). +# * It exercises BOTH ORCID→PubMed resolution paths: as of 2026-07-30, 7 works carry a +# PMID directly and 5 are DOI-only, so `convert_dois_to_pmids` (ID converter, then +# the per-DOI ESearch fallback) really runs. One of the DOI-only entries is a bioRxiv +# preprint that resolves to nothing, which exercises the unresolved branch too. +# * The research has two clearly separated phases — early chromatin-remodelling work, +# then an independent programme on bacterial polyphosphate granules — and measurement +# showed the model recalls the first from pretraining but not the second. That +# separation is what gives T4.1's grounding check something to detect; a subject whose +# entire corpus the model already knows by heart would make it unfalsifiable. +# * Nothing below is pinned to its contents. Every expected value is derived at run +# time from whatever the live record returns. +RACKI = "0000-0003-2209-7301" + + +# --------------------------------------------------------------- grounding vocabulary +# +# Words a wholly invented life-sciences profile would plausibly contain anyway. If a +# corpus term is in here it cannot serve as evidence that the pipeline read its inputs, +# so it is excluded from the derived vocabulary before the check runs. +_GENERIC = { + "analysis", "approach", "approaches", "bacteria", "bacterial", "between", + "biology", "biological", "cellular", "cells", "complex", "complexes", + "computational", "disease", "diseases", "dynamics", "expression", "function", + "functional", "genome", "genomic", "involved", "mechanism", "mechanisms", + "molecular", "process", "processes", "protein", "proteins", "regulates", + "regulation", "research", "researcher", "science", "structural", "structure", + "structures", "studies", "study", "system", "systems", "therapeutic", +} + +_TOKEN = re.compile(r"[A-Za-z][A-Za-z0-9]+") + + +def _stem(word: str) -> str: + """Crude singularisation so 'condensates' and 'condensate' count as one term. + + Deliberately not a real stemmer: a real one would collapse distinct technical terms + and weaken the check. Only a trailing 's' comes off. + """ + w = word.lower() + return w[:-1] if len(w) > 5 and w.endswith("s") and not w.endswith("ss") else w + + +def distinctive_corpus_terms(titles, *, min_docs=2, min_len=7): + """Terms that recur across DISTINCT work titles and are not generic vocabulary. + + Returns ``{term: document_frequency}``. Document frequency, not raw count: a term + repeated inside one title is one paper's worth of evidence, and ORCID routinely + lists the same paper twice (preprint plus version of record), so titles are + de-duplicated first. + """ + seen: set[str] = set() + docs: list[set[str]] = [] + for t in titles: + key = re.sub(r"[^a-z0-9]+", " ", (t or "").lower()).strip() + if not key or key in seen: + continue + seen.add(key) + docs.append({_stem(w) for w in _TOKEN.findall(key)}) + + freq: dict[str, int] = {} + for bag in docs: + for term in bag: + if len(term) >= min_len and term not in _GENERIC: + freq[term] = freq.get(term, 0) + 1 + return {t: n for t, n in freq.items() if n >= min_docs} + + +def mentioned(text: str, terms) -> list[str]: + """Which of ``terms`` occur in ``text`` (substring, case-insensitive on the stem).""" + low = (text or "").lower() + return sorted(t for t in terms if t in low) + + +# A plausible, entirely invented profile for a structural/computational biology lab at +# the same institution. The grounding matcher must find NOTHING in it. If it does, the +# derived vocabulary is too generic and T4.1's control has no teeth — which the test +# says out loud rather than reporting a pass. +_HALLUCINATION_DECOY = ( + "The laboratory investigates the molecular architecture of macromolecular machines " + "using cryo-electron microscopy, single-particle reconstruction and integrative " + "modelling. Recent work has characterised conformational landscapes of membrane " + "transporters and established a computational pipeline for interpreting " + "heterogeneous structural ensembles. The group combines biophysical measurements " + "with molecular dynamics simulation to connect structure to cellular function, and " + "is now extending these approaches to disease-associated variants in human tissue." +) + + +# ------------------------------------------------------------------------- test plumbing + + +class PipelineProbe: + """Observes three seams of `run_profile_pipeline` without replacing any of them. + + Every wrapper delegates to the real function, so the pipeline under test is the real + pipeline making real calls; only the arguments and the call count are recorded. This + is how the LLM-call count (which GM #1 and GM #4 pin) and the synthesis context + (T4.3, T4.4) are observed from the outside. + """ + + def __init__(self): + self.public_calls = 0 + self.private_calls = 0 + self.contexts: list[str] = [] + self.pubs_for_synthesis: list[list[dict]] = [] + + @property + def llm_calls(self) -> int: + return self.public_calls + self.private_calls + + def install(self, monkeypatch): + real_public = profile_pipeline.synthesize_profile + real_private = profile_pipeline.synthesize_private_profile + real_ctx = profile_pipeline._build_synthesis_context + + async def public(context_text, researcher_name): + self.public_calls += 1 + return await real_public(context_text, researcher_name) + + async def private(context_text, researcher_name): + self.private_calls += 1 + return await real_private(context_text, researcher_name) + + def ctx(**kwargs): + out = real_ctx(**kwargs) + self.contexts.append(out) + self.pubs_for_synthesis.append(list(kwargs.get("publications") or [])) + return out + + monkeypatch.setattr(profile_pipeline, "synthesize_profile", public) + monkeypatch.setattr(profile_pipeline, "synthesize_private_profile", private) + monkeypatch.setattr(profile_pipeline, "_build_synthesis_context", ctx) + return self + + +async def seed_pi(db_session, tmp_path, monkeypatch, *, orcid_id=RACKI): + """A User with no name/institution + the AgentRegistry row the revision leg needs. + + The name is left blank on purpose: step 1 of the pipeline fills it from ORCID only + when it is falsy, so a blank name turns "did ORCID actually reach the User row?" into + an observable. + + `PROFILES_DIR` is redirected at a tmp dir so a live run cannot leave a stray + `profiles/public/.md` in the working tree. The export code itself is + untouched — the revision content is still whatever `export_profile_to_markdown` + wrote and read back. + """ + suffix = uuid.uuid4().hex[:8] + monkeypatch.setattr("src.services.profile_export.PROFILES_DIR", tmp_path / "public") + + user = await factories.make_user( + db_session, + name="", + orcid=orcid_id, + institution=None, + department=None, + email=f"t4-{suffix}@example.edu", + onboarding_complete=False, + ) + agent = await factories.make_agent( + db_session, + user=user, + agent_id=f"t4live{suffix}", + bot_name=f"T4Live{suffix}Bot", + pi_name="T4 live subject", + status="pending", + ) + return user, agent + + +async def profile_rows(db_session, user_id) -> list[ResearcherProfile]: + res = await db_session.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == user_id) + ) + return list(res.scalars().all()) + + +async def revisions(db_session, agent_id) -> list[ProfileRevision]: + res = await db_session.execute( + select(ProfileRevision) + .where(ProfileRevision.agent_registry_id == agent_id) + .order_by(ProfileRevision.created_at) + ) + return list(res.scalars().all()) + + +async def publications(db_session, user_id) -> list[Publication]: + res = await db_session.execute( + select(Publication).where(Publication.user_id == user_id) + ) + return list(res.scalars().all()) + + +def as_synthesized(profile: ResearcherProfile) -> dict: + """The dict `_validate_profile` saw, reconstructed from what step 9 stored.""" + return { + "research_summary": profile.research_summary or "", + "techniques": profile.techniques or [], + "experimental_models": profile.experimental_models or [], + "disease_areas": profile.disease_areas or [], + "key_targets": profile.key_targets or [], + "keywords": profile.keywords or [], + } + + +def profile_prose(profile: ResearcherProfile) -> str: + """Every free-text field of the profile, concatenated, for the grounding check.""" + fields = [profile.research_summary or ""] + for lst in ( + profile.techniques, profile.experimental_models, profile.disease_areas, + profile.key_targets, profile.keywords, profile.grant_titles, + ): + fields.extend(lst or []) + return " ".join(fields) + + +# Observations shared between tests. Each live pipeline run costs real money and real +# time, so T4.4 and T4.5 read what T4.1/T4.2 already measured rather than re-running. +# Every consumer states loudly when the producer did not run, so a `-k` selection can +# never turn a missing comparison into a silent pass. +_OBSERVED: dict[str, dict] = {} + + +def require_observation(key: str, producer: str) -> dict: + if key not in _OBSERVED: + pytest.skip( + f"this test compares against the live run recorded by {producer}, which did " + "not run in this session (deselected, or it failed before recording). " + "Nothing was verified — do not read this skip as a pass." + ) + return _OBSERVED[key] + + +# =========================================================================== T4.1 + + +async def test_t41_one_real_orcid_becomes_a_stored_profile_grounded_in_its_works( + db_session, tmp_path, monkeypatch, api_budget +): + """T4.1 — one real ORCID, all the way to a stored profile, over a real database. + + The control is the last block, and it has three layers because the first two are not + enough on their own: + + 1. the vocabulary is derived from a live ORCID fetch the test performs itself, not + from the pipeline's own DB writes, so a pipeline that stored nothing cannot make + the comparison vacuous; + 2. a hand-written decoy profile must score zero, proving the matcher can say no; + 3. and — the layer that actually matters — the same model is asked for a profile of + the same person with an EMPTY publication list, and the real run must produce + corpus vocabulary the name-only run did not. Layers 1 and 2 alone are satisfied + by a wholly hallucinated profile, which is exactly what layer 3 measures. + """ + user, agent = await seed_pi(db_session, tmp_path, monkeypatch) + probe = PipelineProbe().install(monkeypatch) + + api_budget.wait("orcid") + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session) + + # --- the row exists, exactly once, and step 1 reached the User record ------------ + rows = await profile_rows(db_session, user.id) + assert len(rows) == 1, ( + f"{len(rows)} ResearcherProfile rows for one user after one run. " + "researcher_profiles.user_id is UNIQUE, so anything but 1 means the pipeline is " + "writing through a path the constraint does not cover" + ) + assert rows[0].id == profile.id + + await db_session.refresh(user) + assert user.name and user.name.strip(), ( + "the User row still has a blank name after a successful run. Step 1 fills it " + "from ORCID; a blank name means fetch_orcid_profile raised and the pipeline " + "swallowed it (provider down or the person.name shape changed) — every " + "downstream assertion here would then be about an ORCID-less run" + ) + assert user.institution, ( + "ORCID reported no institution for a record whose employment block is populated " + "— fetch_orcid_profile's affiliation-group traversal, or ORCID's shape, changed" + ) + + # --- the synthesized fields ------------------------------------------------------ + assert profile.research_summary and profile.research_summary.strip(), ( + "research_summary is empty after a run that reported success. synthesize_profile " + "raised and the pipeline swallowed it (see the logged 'LLM synthesis failed'): " + "Anthropic is down, the key is bad, or the model's reply did not parse as JSON" + ) + assert isinstance(profile.techniques, list) and len(profile.techniques) >= 3, ( + f"techniques is {profile.techniques!r}; the synthesis prompt requires >=3 and " + "_validate_profile enforces it, so this is a validation bypass, not model drift" + ) + assert isinstance(profile.keywords, list) and profile.keywords, ( + f"keywords is {profile.keywords!r}. T4.1 requires it non-empty. Note that " + "prompts/profile-synthesis.md calls keywords OPTIONAL and _validate_profile does " + "not check it, so an empty list here is a gap between the plan and the prompt, " + "not a pipeline fault" + ) + assert isinstance(profile.disease_areas, list) and profile.disease_areas + assert profile.profile_version == 1, ( + f"profile_version is {profile.profile_version} after the first run; step 9 " + "increments it only when synthesis returned something, so 0 means the fields " + "above came from somewhere else" + ) + assert profile.profile_generated_at is not None + + # T4.1 asks for a non-empty `private_profile_md`. The pipeline NEVER sets that + # column — step 9b writes `private_profile_seed`, and `private_profile_md` is the + # live copy the PI edits later through the web UI. GM #1 pins the same thing + # ('private_profile_md': None in the snapshot). Both halves are asserted so the + # discrepancy is recorded rather than quietly reinterpreted. + assert profile.private_profile_seed and profile.private_profile_seed.strip(), ( + "step 9b produced no private-profile seed. synthesize_private_profile raised " + "(logged as 'Private profile seed generation failed') — same three causes as " + "the public synthesis above" + ) + assert profile.private_profile_md is None, ( + "the pipeline set private_profile_md. It has never done that (step 9b writes " + "private_profile_seed, and GM #1 snapshots private_profile_md as None); if this " + "changed, the PI's hand-edited private profile is now being overwritten by a " + "monthly refresh" + ) + + # --- _validate_profile accepted it, and the row says so --------------------------- + assert profile_pipeline._validate_profile(as_synthesized(profile)) is True, ( + "the profile the pipeline STORED does not pass _validate_profile. Since 0023 " + "step 9 records that verdict in synthesis_validated rather than discarding it, " + "so this should be impossible for a True flag below — a mismatch means the " + "stored fields and the recorded verdict came from different syntheses" + ) + assert profile.synthesis_validated is True, ( + f"synthesis_validated is {profile.synthesis_validated!r} after a run whose " + "stored fields pass the validator. False means step 8's retry also failed and " + "the draft was stored marked (the run cost 2 public calls — see below); None " + "means step 9 stored the fields without recording the verdict, which is the " + "pre-0023 defect back again" + ) + # Grounded, and the row can prove it: this is the assertion that separates a real + # profile from the one T4.4 produces with PubMed unreachable. + assert (profile.evidence_pub_count or 0) > 0, ( + f"evidence_pub_count is {profile.evidence_pub_count!r} after a live run over a " + f"real corpus ({profile.evidence_pmid_count!r} PMIDs in hand). Either no abstract " + "reached the prompt — in which case this whole test is measuring a fabricated " + "profile — or step 9 is not writing the count, and a fabricated profile is " + "indistinguishable from this one again" + ) + assert profile.evidence_state == "grounded", ( + f"evidence_state is {profile.evidence_state!r} for a live run with a real corpus" + ) + assert probe.public_calls == 1, ( + f"{probe.public_calls} public-synthesis calls. 2 means validation rejected the " + "first reply and the stricter retry fired — the profile is still stored, but " + "the run cost double and GM #1's 'exactly two LLM calls' no longer holds" + ) + assert probe.private_calls == 1 + + # --- publications were persisted --------------------------------------------------- + pubs = await publications(db_session, user.id) + assert pubs, ( + "no Publication rows. Either ORCID returned no works (provider/record change) or " + "every PubMed fetch failed (NCBI down or rate-limiting — look for 'Failed to " + "fetch PubMed batch'). The grounding check below cannot mean anything without them" + ) + assert all(p.pmid for p in pubs), "a Publication was stored with no PMID" + assert all(p.title and p.title.strip() for p in pubs), ( + "a Publication was stored with an empty title — _parse_pubmed_xml's ArticleTitle " + "read is broken, or NCBI moved it" + ) + assert len({p.pmid for p in pubs}) == len(pubs), ( + "duplicate PMIDs stored for one user on a single run" + ) + + # --- a revision was recorded --------------------------------------------------------- + revs = await revisions(db_session, agent.id) + assert len(revs) == 1, ( + f"{len(revs)} ProfileRevision rows after one run, expected 1. The revision leg is " + "gated on BOTH an AgentRegistry row and a successful markdown export, so 0 means " + "export_profile_to_markdown returned None (a filesystem problem), not an LLM one" + ) + assert revs[0].profile_type == "public" + assert revs[0].mechanism == "pipeline" + assert profile.research_summary in revs[0].content, ( + "the recorded revision does not contain the summary that was just generated — " + "the revision is snapshotting a stale export" + ) + + # --- THE CONTROL: is the summary grounded in the works that were fetched? ----------- + api_budget.wait("orcid") + live_works = await orcid_service.fetch_orcid_works(RACKI) + assert len(live_works) >= 5, ( + f"ORCID returned {len(live_works)} works for {RACKI}; the grounding check needs a " + "real corpus to derive vocabulary from and proves nothing without one" + ) + corpus = distinctive_corpus_terms([w.get("title", "") for w in live_works]) + assert len(corpus) >= 3, ( + f"only {len(corpus)} distinctive terms could be derived from the live work titles " + f"({sorted(corpus)}). The check below would be near-vacuous; the record's titles " + "have changed character and this test needs a different subject" + ) + + decoy_hits = mentioned(_HALLUCINATION_DECOY, corpus) + assert not decoy_hits, ( + f"the derived vocabulary {sorted(corpus)} also matches a completely invented " + f"profile (on {decoy_hits}). The matcher cannot say no, so the assertion below " + "would pass for a hallucinated summary — tighten _GENERIC or min_len" + ) + + summary_hits = mentioned(profile.research_summary, corpus) + assert summary_hits, ( + "THE GENERATED SUMMARY CONTAINS NOTHING FROM THE FETCHED WORKS. None of the " + f"terms that recur across this researcher's own publication titles ({sorted(corpus)}) " + "appears in it, while the same matcher correctly finds nothing in a decoy. Either " + "the corpus never reached the prompt (check _build_synthesis_context) or the model " + f"ignored it. Summary was: {profile.research_summary[:400]!r}" + ) + whole_hits = mentioned(profile_prose(profile), corpus) + assert len(whole_hits) >= 2, ( + f"the whole profile matches only {whole_hits} of {sorted(corpus)}. One term could " + "be a coincidence; the profile as a whole should reflect more than one recurring " + "theme of a corpus this small" + ) + + # --- and the control that makes the two assertions above mean anything -------------- + # + # The decoy above is hand-written, which only proves the matcher CAN say no. It does + # not prove it would say no to THIS model writing about THIS person. Measured on + # 2026-07-30: asked to profile "Lisa Racki, Scripps Research Institute, Integrative + # Structural and Computational Biology" with an EMPTY publication list, Opus returns a + # confident, _validate_profile-passing profile that already contains "chromatin", + # "remodeling" and "histone" — three of the seven derived corpus terms — purely from + # what it remembers about her. A grounding check that accepted any corpus term would + # therefore pass for a pipeline that fetched nothing at all. + # + # So the control is run empirically, here, against the same model on the same day: + # synthesize once more from a context stripped of every work, and require the real + # run to have said something the name-only run did NOT. That difference is the only + # part of the summary that is attributable to the fetched works rather than to the + # model's prior knowledge of the researcher. + from src.services.llm import synthesize_profile as _raw_synthesize + + nameonly_context = ( + "## Researcher Information\n" + f"- Name: {user.name}\n" + f"- Institution: {user.institution}\n" + f"- Department: {user.department}" + ) + assert not mentioned(nameonly_context, corpus), ( + "the name-only control context already contains corpus vocabulary, so the " + "comparison below would understate grounding" + ) + ungrounded = await _raw_synthesize(nameonly_context, user.name) + ungrounded_hits = mentioned(ungrounded.get("research_summary", ""), corpus) + + evidence = sorted(set(summary_hits) - set(ungrounded_hits)) + assert evidence, ( + "THE PROFILE CANNOT BE SHOWN TO HAVE USED THE FETCHED WORKS. Every corpus term " + f"in the generated summary ({sorted(summary_hits)}) is also produced by the same " + "model given nothing but this researcher's name, institution and department " + f"({sorted(ungrounded_hits)}). A pipeline whose ORCID and PubMed legs were both " + "dead would have produced an equally 'grounded'-looking profile, so this run " + "provides no evidence that the corpus reached the model. Grounded summary: " + f"{profile.research_summary[:300]!r} ... Name-only summary: " + f"{ungrounded.get('research_summary', '')[:300]!r}" + ) + + _OBSERVED["single_run"] = { + "context": probe.contexts[0], + "pubs_for_synthesis": probe.pubs_for_synthesis[0], + "llm_calls": probe.llm_calls, + "profile_version": profile.profile_version, + "research_summary": profile.research_summary, + "raw_abstracts_hash": profile.raw_abstracts_hash, + "private_profile_md": profile.private_profile_md, + "private_profile_seed": profile.private_profile_seed, + "synthesis_validated": profile.synthesis_validated, + "evidence_pmid_count": profile.evidence_pmid_count, + "evidence_pub_count": profile.evidence_pub_count, + "evidence_state": profile.evidence_state, + # Read off the ORM object, NOT off as_synthesized() — that helper coerces None to + # ""/[] for the validator, which would make T4.5's type comparison always pass. + "field_types": { + k: type(getattr(profile, k)).__name__ + for k in ( + "research_summary", "techniques", "experimental_models", + "disease_areas", "key_targets", "keywords", + ) + }, + "pub_count": len(pubs), + # Every column GM #1's snapshot enumerates for a stored publication, so T4.5 can + # reconcile the shape as well as the values. + "stored_pubs": [ + { + "pmid": p.pmid, "doi": p.doi, "title": p.title, "journal": p.journal, + "year": p.year, "abstract": p.abstract, "pmcid": p.pmcid, + } + for p in pubs + ], + "revision_count": len(revs), + "corpus": corpus, + "summary_hits": summary_hits, + "ungrounded_hits": ungrounded_hits, + "grounding_evidence": evidence, + } + + +# =========================================================================== T4.2 + + +async def test_t42_a_second_run_updates_the_same_row_and_adds_a_second_revision( + db_session, tmp_path, monkeypatch, api_budget +): + """T4.2 — idempotency, both halves. + + Half one: the profile row must not be duplicated. Half two: a *second* + ProfileRevision must nonetheless be written — history is append-only, and a + "no duplicates" implementation that also skipped the revision would satisfy half one + while silently losing the audit trail. The publication rows are checked the same + way: updated in place, never re-inserted. + """ + user, agent = await seed_pi(db_session, tmp_path, monkeypatch) + probe = PipelineProbe().install(monkeypatch) + + api_budget.wait("orcid") + first = await profile_pipeline.run_profile_pipeline(user.id, db_session) + first_id = first.id + first_version = first.profile_version + first_seed = first.private_profile_seed + first_pubs = {p.pmid for p in await publications(db_session, user.id)} + first_revs = await revisions(db_session, agent.id) + calls_after_first = probe.llm_calls + + assert first_version == 1 and first_pubs and len(first_revs) == 1, ( + "the first run did not reach a good state, so the second run cannot test " + f"idempotency: version={first_version} pubs={len(first_pubs)} " + f"revisions={len(first_revs)}" + ) + + api_budget.wait("orcid") + second = await profile_pipeline.run_profile_pipeline(user.id, db_session) + + # --- half one: no duplicate profile row ------------------------------------------- + rows = await profile_rows(db_session, user.id) + assert len(rows) == 1, ( + f"{len(rows)} ResearcherProfile rows after two runs — step 6's " + "select-then-create is inserting instead of loading" + ) + assert second.id == first_id, "the second run created a different profile row" + assert second.profile_version == first_version + 1 == 2, ( + f"profile_version went {first_version} -> {second.profile_version}; step 9 " + "increments by one per successful synthesis" + ) + second_pubs = {p.pmid for p in await publications(db_session, user.id)} + assert second_pubs == first_pubs, ( + "the publication set changed between two runs of the same corpus. New PMIDs " + f"({sorted(second_pubs - first_pubs)}) mean the existing-publication lookup " + f"missed; lost PMIDs ({sorted(first_pubs - second_pubs)}) mean a fetch failed" + ) + all_pubs = await publications(db_session, user.id) + assert len(all_pubs) == len(second_pubs), ( + f"{len(all_pubs)} Publication rows for {len(second_pubs)} distinct PMIDs — the " + "second run re-inserted instead of updating" + ) + + # --- half two: a SECOND revision was recorded -------------------------------------- + revs = await revisions(db_session, agent.id) + assert len(revs) == 2, ( + f"{len(revs)} ProfileRevision rows after two runs, expected 2. Profile history is " + "append-only and the monthly refresh depends on it; one row means the second run " + "overwrote history, which is the failure that makes 'what changed?' unanswerable" + ) + assert revs[0].id != revs[1].id, "the same revision row was returned twice" + assert all(r.mechanism == "pipeline" and r.profile_type == "public" for r in revs) + assert second.research_summary in revs[-1].content, ( + "the second revision does not contain the second run's summary" + ) + + # The seed is generated once and then left alone (GM #4 pins this). A pipeline that + # regenerated it every month would silently discard the PI's edits. + assert second.private_profile_seed == first_seed, ( + "the re-run regenerated private_profile_seed. Step 9b is guarded on the seed " + "being absent; if that guard broke, every refresh overwrites the PI's staged text" + ) + + _OBSERVED["rerun"] = { + "first_version": first_version, + "second_version": second.profile_version, + "same_profile_row": second.id == first_id, + "pub_count_after_two_runs": len(all_pubs), + "seed_set_after_first_run": first_seed is not None, + "seed_unchanged_on_rerun": second.private_profile_seed == first_seed, + "llm_calls_total": probe.llm_calls, + "llm_calls_first_run": calls_after_first, + "revision_count": len(revs), + } + + +# =========================================================================== T4.3 + + +async def test_t43_the_synthesis_context_is_bounded_and_contains_the_fetched_works( + api_budget, +): + """T4.3 — `_build_synthesis_context` under a real corpus. + + Spends no Anthropic calls: the context is a pure function of the fetched data, and + the fetched data is the expensive-to-fake part. The pipeline's step 3-4 sequence is + reproduced here rather than observed through a run, so this test stands alone and a + `-k t43` selection still verifies something. + + Two independent properties: + * BOUNDED — it goes verbatim into a prompt, so an unbounded context is a bill and + a context-window overflow. The 30-publication cap is exercised with a padded + corpus, because a live record with 12 works cannot reach it. + * COMPLETE — every publication that survived the research-article filter appears. + Control: the assertion is preceded by a check that there is more than a header + to find, so "all N titles present" cannot be satisfied by N == 0. + """ + api_budget.wait("orcid") + profile = await orcid_service.fetch_orcid_profile(RACKI) + api_budget.wait("orcid") + works = await orcid_service.fetch_orcid_works(RACKI) + assert works, "ORCID returned no works — nothing below would be meaningful" + + pmids = [w["pmid"] for w in works if w.get("pmid")] + doi_only = sorted({w["doi"] for w in works if w.get("doi") and not w.get("pmid")}) + if doi_only: + api_budget.wait("ncbi") + resolved = await pubmed.convert_dois_to_pmids(doi_only) + pmids.extend(resolved.values()) + pmids = sorted(set(pmids)) + assert len(pmids) >= 5, ( + f"only {len(pmids)} PMIDs resolved from {len(works)} ORCID works. Either NCBI is " + "degraded (convert_dois_to_pmids swallows errors and returns {}) or the record " + "changed; a corpus this thin makes the completeness check below weak" + ) + + api_budget.wait("ncbi") + records = await pubmed.fetch_pubmed_records(pmids) + assert records, ( + "efetch returned nothing for PMIDs that exist — NCBI is down or rate-limiting, " + "not a parser fault (fetch_pubmed_records logs 'Failed to fetch PubMed batch')" + ) + + for_synthesis = [ + r for r in records + if r.get("abstract") + and not any( + t in profile_pipeline.EXCLUDED_TYPES + for t in (x.lower() for x in r.get("pub_types", [])) + ) + ] + assert len(for_synthesis) >= 3, ( + f"only {len(for_synthesis)} of {len(records)} records survived the " + "research-article + has-abstract filter; the completeness check needs more" + ) + + context = profile_pipeline._build_synthesis_context( + orcid_profile=profile, + grant_titles=[], + publications=for_synthesis, + methods_by_pmid={}, + ) + + # --- complete ---------------------------------------------------------------------- + assert context.count("\n### ") == len(for_synthesis), ( + f"the context has {context.count(chr(10) + '### ')} publication sections for " + f"{len(for_synthesis)} publications — works are being dropped between step 4 and " + "the prompt, which is exactly how a profile ends up ungrounded" + ) + absent = [p["title"] for p in for_synthesis if p.get("title") and p["title"] not in context] + assert not absent, ( + f"{len(absent)} fetched publication titles never reach the prompt: {absent[:3]}" + ) + with_abstract = sum(1 for p in for_synthesis if p["abstract"][:1500] in context) + assert with_abstract == len(for_synthesis), ( + f"only {with_abstract}/{len(for_synthesis)} abstracts appear in the context — the " + "titles are there but the evidence is not" + ) + assert profile.get("name", "") in context + + # --- bounded ------------------------------------------------------------------------- + # Arithmetic the code commits to: <=30 publications, each abstract truncated at 1500 + # chars, <=10 methods sections truncated at 2000. Header and grant lines are small. + ceiling = 30 * (1500 + 400) + 10 * (2000 + 100) + 4000 + assert len(context) <= ceiling, ( + f"the synthesis context is {len(context)} chars, over the {ceiling}-char ceiling " + "implied by the truncation constants in _build_synthesis_context. One of those " + "truncations has been removed and the whole abstract corpus is now going into " + "every prompt" + ) + + long_abstracts = [p for p in for_synthesis if len(p.get("abstract", "")) > 1500] + if long_abstracts: + p = long_abstracts[0] + assert p["abstract"] not in context, ( + f"the full {len(p['abstract'])}-char abstract for PMID {p.get('pmid')} is in " + "the context — the [:1500] truncation is gone" + ) + assert p["abstract"][:1500] in context + else: + # Not a skip: the rest of the test is unaffected, but the reader should know the + # truncation itself went unexercised on today's data. + assert all(len(p.get("abstract", "")) <= 1500 for p in for_synthesis) + + # The 30-publication cap, exercised with a padded corpus built from the live records. + padded = [] + for i in range(40): + src = dict(for_synthesis[i % len(for_synthesis)]) + src["title"] = f"PAD{i:02d} {src.get('title', '')}" + src["year"] = 2100 - i # strictly descending, so the order is knowable + padded.append(src) + capped = profile_pipeline._build_synthesis_context( + orcid_profile=profile, grant_titles=[], publications=padded, methods_by_pmid={} + ) + assert capped.count("\n### ") == 30, ( + f"a 40-publication corpus produced {capped.count(chr(10) + '### ')} sections; " + "_build_synthesis_context caps at 30 and an uncapped prompt scales with a PI's " + "whole career" + ) + assert "PAD00 " in capped and "PAD29 " in capped, ( + "the 30 kept publications are not the 30 most recent — the sort is broken" + ) + assert "PAD30 " not in capped and "PAD39 " not in capped + + +# =========================================================================== T4.4 + + +def _ncbi_hosts() -> set[str]: + """The hosts every NCBI call in the system goes to, read from the code.""" + return { + urlparse(pubmed.EUTILS_BASE).hostname, + urlparse(pubmed.IDCONV_BASE).hostname, + } + + +async def test_t44_pubmed_unreachable_still_yields_a_profile_but_a_measurably_thinner_one( + db_session, tmp_path, monkeypatch, api_budget +): + """T4.4 — degradation. NCBI unreachable; ORCID and Anthropic still live. + + respx is used *inside* the live tier: every host `src/services/pubmed.py` talks to + raises ConnectError, and everything else — ORCID, api.anthropic.com — passes through + to the real network. That is the honest simulation of "PubMed is down" and it is the + reason this test is here rather than in the contract tier. + + Requirement: onboarding must not fail. Control: the degraded profile must be + measurably thinner, otherwise "it still produced a profile" is satisfied by a + pipeline that never used PubMed in the first place. + + The thinness assertions are deliberately the deterministic ones — publication rows, + the abstract hash, and the size and structure of the synthesis context, which IS the + profile's entire evidence base. Word counts of model prose are not evidence of + anything. + """ + baseline = require_observation("single_run", "T4.1") + user, _agent = await seed_pi(db_session, tmp_path, monkeypatch) + probe = PipelineProbe().install(monkeypatch) + + hosts = _ncbi_hosts() + assert hosts and None not in hosts, f"could not read NCBI hosts from the code: {hosts}" + + api_budget.wait("orcid") + with respx.mock(assert_all_called=False) as router: + for host in hosts: + router.route(host=host).mock( + side_effect=httpx.ConnectError(f"simulated {host} outage (T4.4)") + ) + blocked = [r for r in router.routes] + router.route().pass_through() # ORCID and Anthropic reach the real network + + profile = await profile_pipeline.run_profile_pipeline(user.id, db_session) + + ncbi_attempts = sum(r.call_count for r in blocked) + + # Control on the simulation itself: if the pipeline never tried to reach NCBI, this + # test degraded nothing and its result means nothing. + assert ncbi_attempts > 0, ( + f"the pipeline made no request to any of {sorted(hosts)}, so the simulated outage " + "blocked nothing. Either the URLs moved or PubMed is no longer on this path — " + "either way the 'degraded' profile below is just a normal profile" + ) + + # --- onboarding still completes ---------------------------------------------------- + rows = await profile_rows(db_session, user.id) + assert len(rows) == 1, ( + f"{len(rows)} profile rows with PubMed down; a PubMed outage must not stop a PI " + "being onboarded" + ) + assert profile.research_summary and profile.research_summary.strip(), ( + "with PubMed unreachable the pipeline produced no summary at all. Steps 4 and 5 " + "are individually try/excepted precisely so a PubMed outage degrades rather than " + "aborts; if this fails, one of those handlers stopped catching" + ) + assert profile.profile_version == 1 + await db_session.refresh(user) + assert user.name and user.name.strip(), ( + "ORCID data did not land either — the pass-through route is broken and this test " + "simulated a total outage, not a PubMed one" + ) + + # --- the control: measurably thinner ------------------------------------------------- + degraded_pubs = await publications(db_session, user.id) + assert degraded_pubs == [], ( + f"{len(degraded_pubs)} Publication rows were stored while every NCBI host was " + "unreachable — those rows came from somewhere other than PubMed" + ) + assert baseline["pub_count"] > 0, ( + "the T4.1 baseline stored no publications either, so 'thinner' is not measurable" + ) + assert profile.raw_abstracts_hash == hashlib.sha256(b"").hexdigest(), ( + "the abstract hash is not the hash of an empty corpus, so the pipeline thinks it " + f"synthesized from abstracts it never fetched: {profile.raw_abstracts_hash}" + ) + assert profile.raw_abstracts_hash != baseline["raw_abstracts_hash"] + + degraded_context = probe.contexts[0] + full_context = baseline["context"] + assert "## Publications" not in degraded_context, ( + "the degraded synthesis context still has a Publications section" + ) + assert "## Publications" in full_context, ( + "the T4.1 baseline context had no Publications section either — the comparison " + "below is meaningless" + ) + assert len(degraded_context) < 0.2 * len(full_context), ( + f"the degraded context is {len(degraded_context)} chars against the baseline's " + f"{len(full_context)} — not measurably thinner, so PubMed was contributing almost " + "nothing to the prompt even when it was up" + ) + + # The sharpest statement of the degradation, and the reason this is worth a test: + # with PubMed down the prompt contains none of the researcher's own subject matter, + # because ORCID *works* only enter the context via their PubMed records. Whatever the + # model then writes is not grounded in anything the pipeline fetched. + corpus = baseline["corpus"] + assert not mentioned(degraded_context, corpus), ( + "the degraded context still contains the corpus vocabulary " + f"{mentioned(degraded_context, corpus)}, so ORCID works are reaching the prompt " + "by some path and the claim below would be wrong" + ) + assert mentioned(full_context, corpus), ( + "the baseline context contains none of the corpus vocabulary — the derivation is " + "broken, not the pipeline" + ) + + # Still true, and still worth asserting: the profile synthesized from a name and a + # department passes the same validator as the one synthesized from a dozen abstracts. + # _validate_profile only measures SHAPE — a 150-250 word summary, three techniques, a + # disease area — and a fluent model satisfies all three from prior knowledge. No + # amount of tightening the validator finds this case, which is why the discriminator + # below is a count of evidence and not a quality score. + assert profile_pipeline._validate_profile(as_synthesized(profile)) is True, ( + "the evidence-free profile now FAILS _validate_profile. That is an improvement, " + "not a regression, but it changes the pipeline's behaviour under a PubMed outage " + "(step 8 would retry, then store the draft marked unvalidated) and this test " + "needs updating" + ) + assert profile.synthesis_validated is True, ( + f"synthesis_validated is {profile.synthesis_validated!r}; the assertion above says " + "the stored fields pass the validator, so the recorded verdict disagrees with the " + "validator applied to the same row" + ) + + # What USED to be the finding here: the fabricated profile was stored with the same + # profile_version 1 and no marker of any kind, so nothing downstream — the agent + # prompt builder, the public profile page, the monthly refresh — could tell it from a + # profile grounded in a dozen abstracts. Migration 0023 closed that. The row now + # carries what the synthesis was actually founded on, and this is the live proof of + # it: ORCID gave the pipeline identifiers, PubMed gave it nothing, so the profile is + # ungrounded AND says which of the two ungrounded cases it is. + assert profile.evidence_pub_count == 0, ( + f"evidence_pub_count is {profile.evidence_pub_count!r} while every NCBI host was " + "unreachable. No abstract can have reached the prompt, so a non-zero count means " + "step 9 is recording something other than what it synthesized from" + ) + assert (profile.evidence_pmid_count or 0) > 0, ( + f"evidence_pmid_count is {profile.evidence_pmid_count!r}. ORCID is up in this test " + "and this record carries PMIDs directly (7 of 12 as of 2026-07-30), so zero means " + "the ORCID leg failed too and this is a total outage, not a PubMed one — and the " + "state below would then be 'lost' for the wrong reason" + ) + assert profile.evidence_state == "evidence_lost", ( + f"the fabricated profile reports evidence_state {profile.evidence_state!r}. " + "'no_evidence_available' would be a false negative — it is the answer reserved " + "for a researcher who genuinely has nothing indexed, and it tells an operator " + "NOT to regenerate, which is exactly wrong after an outage" + ) + # The comparison that makes the discriminator meaningful: the grounded baseline and + # this run are the same profile_version, so version cannot separate them and the + # evidence counts must. + assert profile.profile_version == baseline["profile_version"], ( + "the degraded and grounded runs no longer share a profile_version, so the claim " + "that they are indistinguishable without the evidence counts is out of date" + ) + assert baseline["evidence_state"] == "grounded" != profile.evidence_state, ( + f"the grounded baseline reports evidence_state {baseline['evidence_state']!r} and " + f"this ungrounded run reports {profile.evidence_state!r} — the column does not " + "separate the two cases it exists to separate" + ) + + # Thinness at the level of the profile text, not just its evidence base. The degraded + # summary is NOT empty of the researcher's subject matter — the model recognises the + # name — so the measurable claim is a strict subset, not an absence. Measured + # 2026-07-30: the degraded summary reached 2 of the 7 corpus terms (chromatin, + # remodeling) against 7 of 7 for the grounded run. If this ever came out equal, the + # PubMed leg would be contributing nothing the model did not already know. + degraded_hits = set(mentioned(profile.research_summary, corpus)) + baseline_hits = set(baseline["summary_hits"]) + assert degraded_hits < baseline_hits, ( + f"the degraded summary covers {sorted(degraded_hits)} of the corpus vocabulary " + f"and the grounded one covers {sorted(baseline_hits)} — not a strict subset, so " + "losing PubMed entirely cost the profile nothing measurable and the pipeline's " + "whole PubMed leg is decorative for this researcher" + ) + + _OBSERVED["degraded"] = { + "context_len": len(degraded_context), + "baseline_context_len": len(full_context), + "summary": profile.research_summary, + "summary_hits": mentioned(profile.research_summary, corpus), + "baseline_summary_hits": baseline["summary_hits"], + "validated": profile_pipeline._validate_profile(as_synthesized(profile)), + "ncbi_attempts": ncbi_attempts, + } + + +# =========================================================================== T4.5 + + +def _gm_claims() -> dict: + """What the four mocked golden masters assert, restated as checkable claims. + + Read out of the snapshot file rather than retyped, so a `--snapshot-update` that + changed a GM cannot leave this reconciliation silently describing the old one. + """ + path = ( + Path(__file__).resolve().parents[1] + / "characterization" / "__snapshots__" / "test_profile_pipeline_gm.ambr" + ) + return {"text": path.read_text(encoding="utf-8"), "path": path} + + +async def test_t45_the_four_mocked_golden_masters_still_describe_the_live_shape( + db_session, tmp_path, monkeypatch, api_budget +): + """T4.5 — reconcile the live run against the four mocked golden masters. + + The GM suite is entirely fake-driven. Each of its four snapshots makes claims that a + live run can confirm or refute; this test states which claim each one makes, checks + it against live observations, and names the snapshot that is wrong when they differ. + + GM #1 test_profile_pipeline_golden_master + one run -> profile_version 1, private_profile_md None, seed set, the six + synthesized fields are list/str, publications carry + pmid/doi/title/journal/year/pmcid/abstract, raw_abstracts_hash is the sha256 + of the joined abstracts, and exactly two LLM calls. + GM #2 test_profile_pipeline_llm_failure_leaves_fields_unset + synthesis raises -> version stays 0, fields stay None, hash still set. + Reproduced live below with an unauthenticated Anthropic client: a real 401 + from api.anthropic.com, zero tokens. + GM #3 test_profile_pipeline_doi_correction_stores_authoritative + the stored DOI is the one PubMed has on file for that PMID, never an + unverified ORCID candidate. Checked against live esummary. + GM #4 test_profile_pipeline_rerun_increments_version_and_updates_pubs + 1 -> 2, same row, publication count stable, seed unchanged, 3 LLM calls. + """ + single = require_observation("single_run", "T4.1") + rerun = require_observation("rerun", "T4.2") + snap = _gm_claims() + assert "test_profile_pipeline_golden_master" in snap["text"], ( + f"{snap['path']} does not contain the golden master this test reconciles against" + ) + + # --- GM #1 ----------------------------------------------------------------------- + assert single["profile_version"] == 1, ( + "GM #1 snapshots profile_version == 1 after one run; live gave " + f"{single['profile_version']}. GM #1 is wrong (or step 9's increment changed)" + ) + assert "'profile_version': 1," in snap["text"] + assert single["private_profile_md"] is None and "'private_profile_md': None," in snap["text"], ( + "GM #1 snapshots private_profile_md as None. Live disagrees, so GM #1 is wrong " + "about which private column the pipeline writes" + ) + assert single["private_profile_seed"], ( + "GM #1 snapshots a non-empty private_profile_seed; live produced none" + ) + expected_types = { + "research_summary": "str", "techniques": "list", "experimental_models": "list", + "disease_areas": "list", "key_targets": "list", "keywords": "list", + } + assert single["field_types"] == expected_types, ( + "the live profile's field types differ from the ones GM #1's snapshot encodes " + f"(str + five lists): {single['field_types']}. GM #1's _VALID_PROFILE fixture no " + "longer matches what a real model returns through _extract_json" + ) + # GM #1's snapshot enumerates seven columns per stored publication. A live row must + # populate the identifying ones; pmcid and abstract are legitimately null for some + # records (Watson & Crick has neither), so those are checked for presence-of-key only. + gm_pub_keys = {"pmid", "doi", "title", "journal", "year", "pmcid", "abstract"} + for row in single["stored_pubs"]: + assert gm_pub_keys <= set(row), ( + "a live Publication row is missing columns GM #1's snapshot enumerates: " + f"{sorted(gm_pub_keys - set(row))}" + ) + populated = { + k for k in gm_pub_keys + if any(row.get(k) not in (None, "") for row in single["stored_pubs"]) + } + assert populated >= {"pmid", "doi", "title", "journal", "year"}, ( + "GM #1's snapshot shows every publication carrying pmid/doi/title/journal/year. " + f"Across {len(single['stored_pubs'])} live rows only {sorted(populated)} were ever " + "populated, so the GM overstates what the real ingest produces" + ) + # The hash rule, recomputed from the real records the real run synthesized from. + expected_hash = hashlib.sha256( + "\n".join(p.get("abstract", "") for p in single["pubs_for_synthesis"]).encode() + ).hexdigest() + assert single["raw_abstracts_hash"] == expected_hash, ( + "GM #1 pins raw_abstracts_hash as the sha256 of the newline-joined abstracts of " + "the publications passed to synthesis. Recomputing that over the live corpus " + f"gives {expected_hash[:12]}… but the pipeline stored " + f"{single['raw_abstracts_hash'][:12]}… — the hashed set is not the synthesized set" + ) + assert single["llm_calls"] == 2, ( + f"GM #1 asserts exactly two LLM calls on the happy path; the live run made " + f"{single['llm_calls']}. Three means _validate_profile rejected a real model's " + "output and the retry fired — the GM never sees that because its fixture is " + "hand-tuned to pass validation, so the GM understates the real cost per profile" + ) + + # --- GM #4 ----------------------------------------------------------------------- + gm4_expected = { + "first_version": 1, + "second_version": 2, + "same_profile_row": True, + "seed_set_after_first_run": True, + "seed_unchanged_on_rerun": True, + "llm_calls_total": 3, + } + gm4_live = {k: rerun[k] for k in gm4_expected} + assert gm4_live == gm4_expected, ( + "GM #4 (test_profile_pipeline_rerun_increments_version_and_updates_pubs) does not " + f"describe the live rerun. Snapshot claims {gm4_expected}, live measured " + f"{gm4_live}. If llm_calls_total differs, validation is rejecting real model " + "output; anything else is a behaviour change the GM has not been updated for" + ) + assert rerun["pub_count_after_two_runs"] == single["pub_count"], ( + "GM #4 pins the publication count as unchanged across two runs. Live: " + f"{single['pub_count']} after one run, {rerun['pub_count_after_two_runs']} after two" + ) + + # --- GM #3, against live esummary -------------------------------------------------- + # GM #3 pins the mismatch branch with synthetic DOIs. What a live run can check is + # the invariant that branch exists to protect: nothing is persisted that disagrees + # with the DOI PubMed has on file for that exact PMID. + stored = {p["pmid"]: p["doi"] for p in single["stored_pubs"] if p["doi"]} + assert stored, "no stored DOIs to reconcile — GM #3's invariant is untestable here" + api_budget.wait("ncbi") + authoritative = await pubmed.fetch_authoritative_dois(sorted(stored)) + assert authoritative, ( + "esummary returned no authoritative DOIs; NCBI is degraded (the error is " + "swallowed and {} returned), so GM #3 could not be reconciled this run" + ) + disagreements = { + pmid: (doi, authoritative[pmid]) + for pmid, doi in stored.items() + if pmid in authoritative and doi.lower() != authoritative[pmid].lower() + } + assert not disagreements, ( + "the pipeline persisted DOIs that disagree with the DOI PubMed has on file for " + f"the same PMID: {disagreements}. GM #3 asserts reconcile_pub_doi overwrites the " + "candidate with the authoritative value in exactly this situation, so either the " + "gate is no longer running in the pipeline or normalize_doi stopped canonicalising" + ) + assert len(set(stored) & set(authoritative)) >= 3, ( + f"only {len(set(stored) & set(authoritative))} PMIDs could be reconciled against " + "esummary — this leg proved almost nothing" + ) + + # --- GM #2, reproduced against the real API ------------------------------------------ + # Not a fake: the client below talks to api.anthropic.com and is rejected with a real + # 401. That is the only way to see the pipeline's synthesis-failure path with the real + # SDK's exception types, and it costs nothing. + user, _agent = await seed_pi(db_session, tmp_path, monkeypatch) + probe = PipelineProbe().install(monkeypatch) + monkeypatch.setattr( + "src.services.llm.get_anthropic_client", + lambda: anthropic.Anthropic(api_key="sk-ant-t4-deliberately-invalid"), + ) + api_budget.wait("orcid") + failed = await profile_pipeline.run_profile_pipeline(user.id, db_session) + + assert probe.public_calls == 1 and probe.private_calls == 1, ( + "the failure path did not attempt both synthesis calls, so GM #2's shape is not " + f"the one being reconciled: public={probe.public_calls} private={probe.private_calls}" + ) + gm2_expected = { + "profile_version": 0, + "research_summary": None, + "techniques": None, + "disease_areas": None, + "private_profile_seed": None, + "raw_abstracts_hash_is_set": True, + } + gm2_live = { + "profile_version": failed.profile_version, + "research_summary": failed.research_summary, + "techniques": failed.techniques, + "disease_areas": failed.disease_areas, + "private_profile_seed": failed.private_profile_seed, + "raw_abstracts_hash_is_set": failed.raw_abstracts_hash is not None, + } + assert gm2_live == gm2_expected, ( + "GM #2 (test_profile_pipeline_llm_failure_leaves_fields_unset) does not describe " + "what happens when the REAL Anthropic API rejects the call. Snapshot claims " + f"{gm2_expected}, live measured {gm2_live}. GM #2 raises RuntimeError from a fake; " + "if the live shape differs, the pipeline's `except Exception` is not catching what " + "the real SDK raises" + ) + # Control: the failure must be the one we engineered, not a network outage that would + # have produced the same all-None row for a different reason. + assert failed.raw_abstracts_hash != hashlib.sha256(b"").hexdigest(), ( + "the failed run also had an empty abstract corpus, so this reproduced 'PubMed was " + "down' rather than 'the LLM call failed' and GM #2 was not actually reconciled" + ) diff --git a/tests/integration/test_proposal_review.py b/tests/integration/test_proposal_review.py new file mode 100644 index 0000000..fb7f6dc --- /dev/null +++ b/tests/integration/test_proposal_review.py @@ -0,0 +1,1140 @@ +"""T11 — the proposal review loop, from a concluded thread up to (but not through) +the email seam. + +Scope, stated once so the gap stays visible: + +* **In scope.** The engine's thread-conclusion path (`_check_thread_outcome` -> + `_close_thread`) writing a `ThreadDecision`; the agent dashboard rendering it; the + `/review` and `/reopen` endpoints and the `ProposalReview` rows they write; the + private-channel migration the reopen action triggers. +* **Out of scope by instruction.** Everything inside `src/services/email.py` and + `src/services/email_notifications.py` below `send_proposal_notification`: MIME + assembly, the Reply-To / unsubscribe token wiring, and the SES call itself. The one + test that touches the notification path replaces `send_proposal_notification` with a + recording double and says so in every assertion message, so a reader cannot mistake a + green run here for "proposal email is covered". It is not covered. See + `.notes/full-system-test-plan.md` § Global Constraints. + +Dependencies: the database is REAL (the rolled-back `db_session` from +tests/conftest.py). The LLM, Slack and SES are all doubled — and the autouse +`no_outbound_side_effects` fixture below turns any escape into a hard failure rather +than a silent network call. + +**The state machine, as the code actually implements it.** There is no status column +anywhere; "reviewed" is the *existence* of a `ProposalReview` row for +(thread_decision_id, agent_id), and the rating column doubles as the discriminator: + + thread concluded (ThreadDecision.outcome='proposal') -- no review row + -- POST /review rating 1..4 --> decided, terminal for this agent + -- POST /reopen rating 0 --> reopened, ALSO terminal for this agent + (+ collab_private channel, + ThreadDecision.refined_in_channel set) + +Both edges are one-way and mutually exclusive: the endpoints reject any second action +by the same agent (`/review` with 400 "Already reviewed", `/reopen` with a silent +redirect), and a uniqueness constraint backs it at the DB level. The two agents on a +proposal transition independently. `rating=0` is not reachable through `/review` — +the form validates 1..4 — so it is genuinely a reopen sentinel and not a rating. +""" + +import base64 +import html +import json +import uuid +from types import SimpleNamespace + +import boto3 +import pytest +import slack_sdk +from itsdangerous import TimestampSigner +from sqlalchemy import func, select + +from src.agent.agent import Agent +from src.agent.message_log import LogEntry +from src.agent.simulation import SimulationEngine +from src.agent.state import ThreadState +from src.config import get_settings +from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + EmailEngagementTracker, + EmailNotification, + PrivateChannelMember, + ProposalReview, + ThreadDecision, +) +from src.visibility import VISIBILITY_COLLAB_PRIVATE +from tests import factories +from tests.fakes import FakeSlackClient + +pytestmark = pytest.mark.integration + + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + + +class _FixtureSessionFactory: + """Route a self-opened session (the engine's, the notification worker's) at the + rolled-back test session. + + Both `SimulationEngine._close_thread` and `check_and_send_notifications` do + ``async with self.session_factory() as db: ...; await db.commit()``. The test + session runs in ``create_savepoint`` mode, so that commit only releases a savepoint + and the outer transaction still rolls back at teardown. ``__aexit__`` must NOT close + the fixture-owned session. Same shape as tests/integration/test_message_persistence.py. + """ + + def __init__(self, session): + self._s = session + + def __call__(self): + return self + + async def __aenter__(self): + return self._s + + async def __aexit__(self, *exc): + return False + + +def _auth(user_id) -> dict: + """Forge the signed session cookie SessionMiddleware would issue.""" + signer = TimestampSigner(get_settings().secret_key) + data = base64.b64encode(json.dumps({"user_id": str(user_id)}).encode()) + return {"Cookie": f"copi-session={signer.sign(data).decode()}"} + + +@pytest.fixture(autouse=True) +def no_outbound_side_effects(monkeypatch): + """Belt and braces: nothing in this module may reach SES or Slack. + + Individual tests install their own doubles at a higher level; this exists so that a + *missed* seam cannot quietly send mail to a real inbox or create a channel in the + workspace another agent owns. Verified armed: with the recorder in + `test_the_proposal_notification_is_addressed_...` removed, the real path reaches + ``boto3.client('ses', region_name=...)`` and hits this. + + Caveat on the loudness, so nobody over-trusts it: `send_proposal_notification` + wraps its SES call in a bare ``except Exception``, so on that particular path the + AssertionError is swallowed and logged rather than failing the test. The *block* + still holds — no SES client is ever constructed — but the thing that actually + fails a test when a seam is missed is the recorder assertion, not this fixture. + """ + + def _no_ses(*args, **kwargs): + raise AssertionError( + "boto3.client() was called from a T11 test. Email delivery is out of " + "scope for this plan and must never be exercised — see the module " + f"docstring. args={args!r} kwargs={kwargs!r}" + ) + + def _no_slack(*args, **kwargs): + raise AssertionError( + "slack_sdk.WebClient() was constructed from a T11 test. The live " + "workspace belongs to another agent; every Slack seam here must be " + "doubled." + ) + + monkeypatch.setattr(boto3, "client", _no_ses) + monkeypatch.setattr(slack_sdk, "WebClient", _no_slack) + monkeypatch.setattr("src.agent.slack_client.WebClient", _no_slack) + + +@pytest.fixture +def llm(monkeypatch): + """Double for the only LLM call the conclusion path makes (working-memory + synthesis, `simulation._update_agent_memory`). + + Returns "" so the caller's `if not response: return` short-circuits before it + writes a memory file to disk. The returned list is the evidence that the double was + actually installed on the path under test. + """ + calls: list[dict] = [] + + async def _fake(*args, **kwargs): + calls.append(kwargs) + return "" + + monkeypatch.setattr("src.agent.simulation.generate_agent_response", _fake) + return calls + + +@pytest.fixture +async def lab(db_session): + """Two active agents, each owned by a PI with an email address, plus a run. + + ``alpha``/``beta`` are deliberately not real roster ids: `slack_tokens.env_token` + falls back to `Settings.get_slack_tokens()`, which is keyed by real agent ids, so a + test agent named ``su`` could pick up a production token and flip Slack on. + """ + run = await factories.make_simulation_run(db_session) + pi_a = await factories.make_user( + db_session, name="Ada Alpha", email="ada.alpha@lab.test" + ) + pi_b = await factories.make_user( + db_session, name="Bo Beta", email="bo.beta@lab.test" + ) + reg_a = await factories.make_agent( + db_session, user=pi_a, agent_id="alpha", bot_name="AlphaBot", + pi_name="Ada Alpha", status="active", + ) + reg_b = await factories.make_agent( + db_session, user=pi_b, agent_id="beta", bot_name="BetaBot", + pi_name="Bo Beta", status="active", + ) + await db_session.flush() + return SimpleNamespace( + run_id=run.id, + pi_a_id=pi_a.id, pi_a_email=pi_a.email, pi_a_name=pi_a.name, + pi_b_id=pi_b.id, pi_b_email=pi_b.email, + reg_a_id=reg_a.id, reg_b_id=reg_b.id, + ) + + +def _marker() -> str: + """A token that cannot appear anywhere else in the rendered page.""" + return f"PROPOSALBODY{uuid.uuid4().hex[:10].upper()}" + + +async def _conclude_thread( + db_session, lab, llm_calls, *, channel: str, outcome: str, body: str, +) -> str: + """Drive the REAL conclusion path and return the thread_id. + + Not a factory call: the point of this task's first bullet is that a concluded + thread produces the decision row, so the row has to come out of + `_check_thread_outcome`. ``outcome='proposal'`` replays the :memo:-Summary -> ✅ + handshake; ``outcome='no_proposal'`` replays the ⏸️ close. + """ + agents = [ + Agent(agent_id="alpha", bot_name="AlphaBot", pi_name="Ada Alpha"), + Agent(agent_id="beta", bot_name="BetaBot", pi_name="Bo Beta"), + ] + engine = SimulationEngine( + agents=agents, + slack_clients={}, + session_factory=_FixtureSessionFactory(db_session), + simulation_run_id=lab.run_id, + ) + root_ts = f"{1_700_000_000 + len(channel) * 7 + abs(hash(channel)) % 9000}.000100" + engine.message_log.append(LogEntry( + ts=root_ts, channel=channel, sender_agent_id="beta", sender_name="BetaBot", + content="Opening the discussion.", posted_at=float(root_ts), + )) + thread = ThreadState(thread_id=root_ts, channel=channel, other_agent_id="beta") + agents[0].state.active_threads[root_ts] = thread + + if outcome == "proposal": + summary = f":memo: **Summary — Joint programme**\n\n{body}" + engine.message_log.append(LogEntry( + ts=f"{float(root_ts) + 1:.6f}", channel=channel, sender_agent_id="beta", + sender_name="BetaBot", content=summary, thread_ts=root_ts, + posted_at=float(root_ts) + 1, + )) + await engine._check_thread_outcome(agents[0], thread, "✅ Agreed, let's do it.") + else: + engine.message_log.append(LogEntry( + ts=f"{float(root_ts) + 1:.6f}", channel=channel, sender_agent_id="beta", + sender_name="BetaBot", content=body, thread_ts=root_ts, + posted_at=float(root_ts) + 1, + )) + await engine._check_thread_outcome( + agents[0], thread, f"⏸️ No viable overlap. {body}", + ) + + assert llm_calls, ( + "the working-memory synthesis never ran, so the conclusion path was not " + "actually driven end to end (or the LLM double was installed on the wrong " + "module and a real Anthropic call was attempted)" + ) + db_session.expire_all() + return root_ts + + +async def _decision(db_session, thread_id: str) -> ThreadDecision: + return (await db_session.execute( + select(ThreadDecision).where(ThreadDecision.thread_id == thread_id) + )).scalar_one() + + +@pytest.fixture +async def proposal(db_session, lab, llm): + """One concluded PROPOSAL thread, produced by the engine, ready to review.""" + body = _marker() + thread_id = await _conclude_thread( + db_session, lab, llm, channel="degrader-chem", outcome="proposal", body=body, + ) + td = await _decision(db_session, thread_id) + return SimpleNamespace(id=td.id, thread_id=thread_id, body=body, + channel="degrader-chem") + + +# --------------------------------------------------------------------------- +# 1. A concluded thread produces the decision the review loop consumes +# --------------------------------------------------------------------------- + + +async def test_a_concluded_thread_records_a_proposal_decision(db_session, lab, llm): + """The ✅-confirms-:memo: handshake writes a ThreadDecision with outcome='proposal' + and the summary text starting at the :memo: marker. + + Control: the SAME engine, same session, driven with ⏸️ instead, writes + outcome='no_proposal'. Without it, `outcome == 'proposal'` would also be satisfied + by a `_close_thread` that hard-coded the value. + """ + yes_body, no_body = _marker(), _marker() + yes_ts = await _conclude_thread( + db_session, lab, llm, channel="degrader-chem", outcome="proposal", body=yes_body, + ) + no_ts = await _conclude_thread( + db_session, lab, llm, channel="cold-lead", outcome="no_proposal", body=no_body, + ) + + yes = await _decision(db_session, yes_ts) + no = await _decision(db_session, no_ts) + + assert yes.outcome == "proposal", ( + f"the ✅/:memo: handshake did not close the thread as a proposal: {yes.outcome}" + ) + assert no.outcome == "no_proposal", ( + "the ⏸️ control also came back as 'proposal', so outcome is not being derived " + f"from the conversation at all: {no.outcome}" + ) + assert yes.summary_text.startswith(":memo:"), ( + f"the summary was not extracted from the :memo: marker: {yes.summary_text!r}" + ) + assert yes_body in yes.summary_text + assert {yes.agent_a, yes.agent_b} == {"alpha", "beta"} + assert yes.origin_visibility == "public" + assert yes.refined_in_channel is None, ( + "a freshly concluded thread must not already point at a refinement channel" + ) + + # No ProposalReview exists yet. This is the state machine's real entry point: the + # review row is created by the PI's action, never by the thread concluding. + assert (await db_session.scalar( + select(func.count(ProposalReview.id)).where( + ProposalReview.thread_decision_id.in_([yes.id, no.id]) + ) + )) == 0, ( + "a ProposalReview row appeared without any PI action — concluding a thread is " + "supposed to leave the proposal UNREVIEWED and waiting" + ) + + +# --------------------------------------------------------------------------- +# 2. The dashboard renders it +# --------------------------------------------------------------------------- + + +async def test_the_dashboard_renders_the_proposal_with_a_review_form( + client, db_session, lab, llm, proposal, +): + """The PI's dashboard lists the concluded proposal, with the rate form and the + reopen form pointed at this proposal's id. + + Control for the absence assertion: a `no_proposal` decision in the same run is NOT + listed. Asserting only "the proposal is on the page" would also pass for a + dashboard that listed every thread_decision regardless of outcome. + """ + hidden = _marker() + await _conclude_thread( + db_session, lab, llm, channel="cold-lead", outcome="no_proposal", body=hidden, + ) + + r = await client.get("/agent/alpha/dashboard", headers=_auth(lab.pi_a_id)) + assert r.status_code == 200, r.text[:400] + page = r.text + + assert html.escape(proposal.body, quote=True) in page, ( + "the proposal summary is not on the dashboard the PI is asked to review from" + ) + assert hidden not in page, ( + "a thread that concluded WITHOUT a proposal is being offered for review — the " + "dashboard is listing every ThreadDecision, not just outcome='proposal'" + ) + assert f'action="/agent/alpha/proposals/{proposal.id}/review"' in page, ( + "no review form for this proposal — the PI has no way to act" + ) + assert f'action="/agent/alpha/proposals/{proposal.id}/reopen"' in page, ( + "no reopen form for an ACTIVE agent" + ) + assert "Proposals Awaiting Your Review" in page + + +async def test_the_dashboard_is_not_another_pis_review_surface( + client, db_session, lab, proposal, +): + """Authorization, asserted as a pair: the owning PI gets 200, an unrelated + logged-in user gets 403 on the same URL.""" + outsider = await factories.make_user(db_session, email="nosy@lab.test") + await db_session.flush() + + owner = await client.get("/agent/alpha/dashboard", headers=_auth(lab.pi_a_id)) + other = await client.get("/agent/alpha/dashboard", headers=_auth(outsider.id)) + assert owner.status_code == 200 + assert other.status_code == 403, ( + f"a user with no relationship to agent 'alpha' reached its dashboard: " + f"{other.status_code}" + ) + + +# --------------------------------------------------------------------------- +# 3. The transitions +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "rating,label", + [(1, "Not a good idea"), (4, "Excellent idea")], + ids=["reject", "approve"], +) +async def test_rating_transitions_the_proposal_to_reviewed( + client, db_session, lab, proposal, rating, label, +): + """Approve (4) and reject (1) are the same transition with a different payload: + unreviewed -> reviewed, recorded as one ProposalReview row. + + Both directions are asserted on the DB row AND on the re-rendered page, because a + row written but never surfaced would leave the PI staring at a proposal they have + already decided. + """ + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": str(rating), "comment": f"{label} — because reasons."}, + headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 302, r.text[:400] + assert r.headers["location"] == "/agent/alpha/dashboard" + + db_session.expire_all() + review = (await db_session.execute( + select(ProposalReview).where(ProposalReview.thread_decision_id == proposal.id) + )).scalar_one() + assert review.rating == rating + assert review.agent_id == "alpha" + assert review.user_id == lab.pi_a_id, "the row must be attributed to the PI" + assert review.reviewed_by_user_id == lab.pi_a_id + assert review.delegate_user_id is None, ( + "the PI acted in person; delegate_user_id is only for a delegate's review" + ) + assert review.submitted_via == "web" + assert review.comment == f"{label} — because reasons." + + page = (await client.get( + "/agent/alpha/dashboard", headers=_auth(lab.pi_a_id))).text + assert f"Rating: {rating}/4" in page, ( + "the decided proposal is not shown with its rating in the Reviewed section" + ) + assert f'action="/agent/alpha/proposals/{proposal.id}/review"' not in page, ( + "the review form is still on the page after the proposal was decided — the " + "dashboard has not moved it out of the awaiting-review list" + ) + + +async def test_out_of_range_ratings_are_rejected_and_write_nothing( + client, db_session, lab, proposal, +): + """0 and 5 are refused with 400 and leave no row. + + Positive control in the same test: 3 is accepted and DOES write a row, so "no row" + is not the answer this endpoint gives to everything. 0 matters specifically — + `reopen_proposal` writes rating=0 as its sentinel, and the rating form must not be + able to mint that state directly. + """ + for bad in ("0", "5", "-1"): + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": bad, "comment": ""}, headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 400, f"rating={bad} was not rejected: {r.status_code}" + + db_session.expire_all() + assert (await db_session.scalar( + select(func.count(ProposalReview.id)).where( + ProposalReview.thread_decision_id == proposal.id + ) + )) == 0, "a rejected rating still wrote a ProposalReview row" + + ok = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "3", "comment": ""}, headers=_auth(lab.pi_a_id), + ) + assert ok.status_code == 302 + db_session.expire_all() + assert (await db_session.scalar( + select(func.count(ProposalReview.id)).where( + ProposalReview.thread_decision_id == proposal.id + ) + )) == 1, "the in-range control did not write a row either — the endpoint is broken" + + +async def test_a_decided_review_cannot_be_re_decided( + client, db_session, lab, proposal, +): + """The control the task asks for: reviewed is terminal for that agent. + + Positive control: the OTHER agent in the same proposal can still review it. The + lock is per (thread_decision, agent), not "this proposal is now closed to + everyone" — without this half, a 400 from a globally broken endpoint would look + like correct idempotency. + """ + first = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "4", "comment": "first word"}, headers=_auth(lab.pi_a_id), + ) + assert first.status_code == 302 + + second = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "1", "comment": "changed my mind"}, headers=_auth(lab.pi_a_id), + ) + assert second.status_code == 400, ( + f"a second review by the same agent was accepted ({second.status_code}) — the " + "PI's decision is overwritable" + ) + assert "Already reviewed" in second.text + + db_session.expire_all() + rows = (await db_session.execute( + select(ProposalReview).where(ProposalReview.thread_decision_id == proposal.id) + )).scalars().all() + assert len(rows) == 1, f"the rejected re-review still wrote a row: {rows}" + assert rows[0].rating == 4 and rows[0].comment == "first word", ( + "the first decision was mutated by the rejected second attempt" + ) + + # Positive control — the other side of the proposal is still open. + other = await client.post( + f"/agent/beta/proposals/{proposal.id}/review", + data={"rating": "2", "comment": "from the other lab"}, + headers=_auth(lab.pi_b_id), + ) + assert other.status_code == 302, ( + f"the other agent's PI was ALSO refused ({other.status_code}), so the 400 " + "above is not evidence of a per-agent lock" + ) + db_session.expire_all() + assert sorted(r.rating for r in (await db_session.execute( + select(ProposalReview).where(ProposalReview.thread_decision_id == proposal.id) + )).scalars().all()) == [2, 4] + + +async def test_a_review_cannot_be_filed_against_someone_elses_proposal( + client, db_session, lab, llm, proposal, +): + """An agent that is not a participant is refused, and files nothing. + + Control: the participating agent's PI succeeds on the very same proposal id. + """ + stranger_user = await factories.make_user(db_session, email="gamma@lab.test") + await factories.make_agent( + db_session, user=stranger_user, agent_id="gamma", bot_name="GammaBot", + pi_name="Gia Gamma", status="active", + ) + await db_session.flush() + + bad = await client.post( + f"/agent/gamma/proposals/{proposal.id}/review", + data={"rating": "4", "comment": ""}, headers=_auth(stranger_user.id), + ) + assert bad.status_code == 403, ( + f"an agent that never took part in the thread reviewed it: {bad.status_code}" + ) + db_session.expire_all() + assert (await db_session.scalar( + select(func.count(ProposalReview.id)).where( + ProposalReview.agent_id == "gamma" + ) + )) == 0 + + good = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "4", "comment": ""}, headers=_auth(lab.pi_a_id), + ) + assert good.status_code == 302, ( + "the participant control was refused too, so the 403 above proves nothing" + ) + + +# --------------------------------------------------------------------------- +# 4. The email seam +# --------------------------------------------------------------------------- + + +async def test_the_proposal_notification_is_addressed_but_the_delivery_leg_is_untested( + db_session, lab, proposal, monkeypatch, +): + """THE EMAIL SEAM. Read this before trusting the coverage. + + What this asserts: the notification worker's real selection logic — which users are + eligible, which proposals count as unreviewed, and how the backlog is counted — + reaches `send_proposal_notification` with the right recipient and the right + proposal, for BOTH PIs on the proposal. + + What this deliberately does NOT assert, and what therefore has NO test anywhere in + this plan: the subject line, the text/HTML bodies, the `review+@…` Reply-To + that the inbound reply path depends on, the unsubscribe token, the + `EmailNotification` row `send_proposal_notification` writes, and the SES call. All + of that lives in `src/services/email_notifications.py`, which the plan excludes by + instruction. `send_proposal_notification` is replaced wholesale by the recorder + below, so none of it runs. A green result here means "the system decided to notify + the right people about the right thing", NOT "the email is correct" and NOT "the + email was sent". + + Control: the same call, made again after the PIs have reviewed, records nothing. + Without it, a recorder that fired for every user in the database would look + identical. + """ + from src.services import email_notifications as en + + # Hermetic: this test's premise is an unrestricted outbound allowlist (the + # field's own default, "" = send to everyone) — the fictitious lab.pi_*_email + # addresses must reach _process_user_notifications' is_allowed_recipient + # check. Pin it rather than inherit the deployed .env's + # OUTBOUND_EMAIL_ALLOWLIST on this host, which would otherwise suppress + # both sends and turn `sent` into 0. + monkeypatch.setattr(get_settings(), "outbound_email_allowlist", "") + + recorded: list[dict] = [] + + async def _recording_double( + *, user, thread_decision, agent, other_bot_name, total_unreviewed, db, + ): + recorded.append({ + "to": user.email, + "thread_decision_id": thread_decision.id, + "agent_id": agent.agent_id, + "bot_name": agent.bot_name, + "other_bot_name": other_bot_name, + "total_unreviewed": total_unreviewed, + }) + return True + + monkeypatch.setattr(en, "send_proposal_notification", _recording_double) + + sent = await en.check_and_send_notifications(_FixtureSessionFactory(db_session)) + + assert sent == 2, ( + "expected the notification worker to decide to notify BOTH PIs on this " + f"proposal; it decided to notify {sent}. (Nothing was delivered either way — " + "the SES leg is stubbed out and out of scope.)" + ) + by_recipient = {r["to"]: r for r in recorded} + assert set(by_recipient) == {lab.pi_a_email, lab.pi_b_email}, ( + "wrong recipients queued for the proposal notification: " + f"{sorted(by_recipient)}. NOTE: no message was composed or sent — this " + "asserts only the addressing decision, which is the last thing before the " + "excluded email module." + ) + for email_addr, expect_bot, expect_other in ( + (lab.pi_a_email, "AlphaBot", "BetaBot"), + (lab.pi_b_email, "BetaBot", "AlphaBot"), + ): + call = by_recipient[email_addr] + assert call["thread_decision_id"] == proposal.id, ( + f"{email_addr} would be told about the wrong proposal" + ) + assert call["bot_name"] == expect_bot and call["other_bot_name"] == expect_other, ( + f"{email_addr}'s notification names the wrong pair of bots: {call}. The " + "rendered subject/body that would carry these names is NOT asserted here " + "— composition is inside the excluded module." + ) + assert call["total_unreviewed"] == 1, ( + f"backlog count wrong for {email_addr}: {call['total_unreviewed']}" + ) + + # --- Control: once reviewed, the same worker queues nothing ------------- + for agent_id, pi_id in (("alpha", lab.pi_a_id), ("beta", lab.pi_b_id)): + db_session.add(ProposalReview( + thread_decision_id=proposal.id, agent_id=agent_id, user_id=pi_id, + reviewed_by_user_id=pi_id, rating=3, submitted_via="web", + )) + # The double returned True, so `_process_user_notifications` advanced each + # tracker's send clock. Rewind it, or the second pass would be skipped by the + # weekly frequency gate and the control would pass for the wrong reason. + for tracker in (await db_session.execute( + select(EmailEngagementTracker) + )).scalars().all(): + tracker.last_notification_sent_at = None + await db_session.flush() + recorded.clear() + + again = await en.check_and_send_notifications(_FixtureSessionFactory(db_session)) + assert again == 0 and recorded == [], ( + "a reviewed proposal is still being queued for a reminder email: " + f"{recorded}" + ) + + +async def test_reviewing_on_the_web_retires_the_outstanding_email_notification( + client, db_session, lab, llm, proposal, +): + """The DB half of the email loop, which is on OUR side of the seam: submitting a + web review flips the outstanding `EmailNotification` to 'responded' so the worker + stops nagging. No message is composed, sent, or parsed here. + + Control: an outstanding notification for a DIFFERENT proposal, same user, is left + alone — otherwise `mark_notification_responded` clearing everything unconditionally + would pass. + """ + other_ts = await _conclude_thread( + db_session, lab, llm, channel="second-topic", outcome="proposal", + body=_marker(), + ) + other_td = await _decision(db_session, other_ts) + + mine = EmailNotification( + user_id=lab.pi_a_id, thread_decision_id=proposal.id, + agent_registry_id=lab.reg_a_id, reply_token=f"tok-{uuid.uuid4().hex}", + category="proposal_review", status="sent", + ) + untouched = EmailNotification( + user_id=lab.pi_a_id, thread_decision_id=other_td.id, + agent_registry_id=lab.reg_a_id, reply_token=f"tok-{uuid.uuid4().hex}", + category="proposal_review", status="sent", + ) + db_session.add_all([mine, untouched]) + await db_session.flush() + mine_id, untouched_id = mine.id, untouched.id + + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "3", "comment": ""}, headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 302 + + db_session.expire_all() + after = { + n.id: n for n in (await db_session.execute(select(EmailNotification))) + .scalars().all() + } + assert after[mine_id].status == "responded", ( + "the outstanding reminder for the proposal the PI just reviewed is still " + "'sent' — the worker will keep emailing about a decided proposal" + ) + assert after[mine_id].response_type == "review" + assert after[mine_id].responded_at is not None + assert after[untouched_id].status == "sent", ( + "reviewing one proposal retired the reminder for an unrelated one" + ) + + tracker = (await db_session.execute( + select(EmailEngagementTracker).where( + EmailEngagementTracker.user_id == lab.pi_a_id + ) + )).scalar_one_or_none() + if tracker is not None: + assert tracker.consecutive_missed == 0 + + +# --------------------------------------------------------------------------- +# 5. Reopen -> private channel +# --------------------------------------------------------------------------- + + +@pytest.fixture +def slack_off(monkeypatch): + """Force the migration down its DB-only path. + + `_slack_enabled_for_migration` auto-detects from bot tokens. Our agents have none, + so it would already choose the offline path — but pinning it makes the test's + intent explicit and immune to a stray token appearing in the environment. + """ + async def _off(*args, **kwargs): + return False + + monkeypatch.setattr( + "src.services.private_channels._slack_enabled_for_migration", _off, + ) + + +@pytest.fixture +def slack_on(monkeypatch): + """Force the Slack migration path with a recording fake in place of the real + client. Returns the list of fakes that were constructed.""" + made: list[FakeSlackClient] = [] + + async def _on(*args, **kwargs): + return True + + async def _token(db, agent_id): + return f"xoxb-fake-{agent_id}" + + def _client(agent_id, bot_token): + c = FakeSlackClient(agent_id=agent_id, bot_token=bot_token) + made.append(c) + return c + + monkeypatch.setattr( + "src.services.private_channels._slack_enabled_for_migration", _on) + monkeypatch.setattr( + "src.services.private_channels._get_or_fail_bot_token", _token) + monkeypatch.setattr("src.services.private_channels._make_client", _client) + return made + + +async def test_reopen_opens_the_private_channel_and_files_the_review_together( + client, db_session, lab, proposal, slack_off, +): + """The wiring assertion the task asks for: ONE request produces BOTH the + collab_private channel (with its members and handover) and the rating=0 + ProposalReview that marks the proposal acted-on, and it points the decision at the + new channel. + + They share a transaction on purpose — `migrate_public_thread_to_private` adds rows + to the caller's session and leaves the commit to the reopen endpoint (see the + comment in tests/integration/test_slack_private_live.py). If they ever stop + committing together, one of these two halves disappears. + """ + guidance = "Nail down the ternary-complex geometry before any chemistry." + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": guidance}, headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 302, r.text[:400] + + db_session.expire_all() + channels = (await db_session.execute( + select(AgentChannel).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ) + )).scalars().all() + assert len(channels) == 1, ( + f"expected exactly one private refinement channel, got {len(channels)}" + ) + ch = channels[0] + assert ch.created_by_agent == "alpha" + assert ch.migrated_from_channel_id == f"local:{proposal.channel}", ( + f"the new channel does not record where it came from: " + f"{ch.migrated_from_channel_id}" + ) + assert "alpha" in ch.channel_name and "beta" in ch.channel_name + + td = await _decision(db_session, proposal.thread_id) + assert td.refined_in_channel == ch.channel_id, ( + "the proposal was migrated but the decision row still does not point at the " + f"refinement channel: {td.refined_in_channel!r} != {ch.channel_id!r}" + ) + + review = (await db_session.execute( + select(ProposalReview).where(ProposalReview.thread_decision_id == proposal.id) + )).scalar_one() + assert review.rating == 0, ( + f"reopen is supposed to file the rating=0 sentinel, got {review.rating}" + ) + assert review.comment.startswith("[Reopened] "), review.comment + assert guidance in review.comment + assert review.user_id == lab.pi_a_id + + members = (await db_session.execute( + select(PrivateChannelMember).where( + PrivateChannelMember.agent_channel_id == ch.id + ) + )).scalars().all() + assert {m.agent_id for m in members if m.agent_id} == {"alpha", "beta"} + assert [m.user_id for m in members if m.user_id] == [lab.pi_a_id], ( + "the triggering PI is not a member of the channel that holds their guidance" + ) + + handover = (await db_session.execute( + select(AgentMessage).where(AgentMessage.channel_id == ch.channel_id) + )).scalars().all() + assert any(guidance in (m.content or "") for m in handover), ( + "the PI's guidance never reached the private channel's message history" + ) + assert all(m.visibility == VISIBILITY_COLLAB_PRIVATE for m in handover) + + origin_rows = (await db_session.execute( + select(AgentMessage).where(AgentMessage.channel_name == proposal.channel) + )).scalars().all() + assert origin_rows, "the public origin thread was left with no closing marker" + assert not any(guidance in (m.content or "") for m in origin_rows), ( + "the PI's private guidance was echoed into the PUBLIC origin thread" + ) + + # Observed behaviour, pinned because it is surprising rather than because it is + # right: the rating=0 sentinel puts the reopened proposal in the dashboard's + # "Reviewed Proposals" section labelled "Rating: 0/4" — on a scale the form only + # offers 1..4 on. The PI sees a rating they never gave, and the proposal is no + # longer rateable. Reported as a finding, not fixed here. + page = (await client.get( + "/agent/alpha/dashboard", headers=_auth(lab.pi_a_id))).text + assert "Rating: 0/4" in page, ( + "reopen no longer renders the rating=0 sentinel as a rating — if this was " + "fixed deliberately, update this assertion; the finding is in the T11 report" + ) + assert f'action="/agent/alpha/proposals/{proposal.id}/review"' not in page, ( + "the proposal is still rateable after being reopened for refinement" + ) + + +async def test_a_rating_never_opens_a_private_channel( + client, db_session, lab, proposal, slack_off, +): + """FINDING, pinned as a test. Approving a proposal does NOT trigger the + private-channel reopen — rating and reopen are two separate PI actions behind two + separate endpoints, and only `/reopen` migrates. See the module report. + + Control: the identical setup, driven through `/reopen` instead, DOES create the + channel — so "no channel" is a fact about the rating action, not about a migration + that cannot run in this fixture. + """ + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "4", "comment": "approved"}, headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 302 + db_session.expire_all() + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 0, "a plain rating opened a private refinement channel" + td = await _decision(db_session, proposal.thread_id) + assert td.refined_in_channel is None + + # Control: the reopen action on the same proposal, from the other side. + r2 = await client.post( + f"/agent/beta/proposals/{proposal.id}/reopen", + data={"guidance": "Try the orthogonal readout."}, headers=_auth(lab.pi_b_id), + ) + assert r2.status_code == 302, r2.text[:400] + db_session.expire_all() + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 1, ( + "the /reopen control did not create a channel either, so the assertion above " + "proves nothing about the rating action" + ) + + +async def test_a_rated_proposal_cannot_then_be_reopened_by_the_same_agent( + client, db_session, lab, proposal, slack_off, +): + """The two edges out of "awaiting review" are mutually exclusive. Once alpha has + rated, alpha's reopen is swallowed by the same guard that catches a replayed POST + — note it is a silent 302, not an error, so the PI gets no feedback that their + guidance was discarded (observed behaviour, reported). + + Control: beta, which has not acted, CAN still reopen the same proposal — so "no + channel" is a fact about alpha's spent transition, not about the migration being + unavailable in this fixture. + """ + rated = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "3", "comment": "decided"}, headers=_auth(lab.pi_a_id), + ) + assert rated.status_code == 302 + + swallowed = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": "Actually, refine it instead."}, + headers=_auth(lab.pi_a_id), + ) + assert swallowed.status_code == 302 + db_session.expire_all() + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 0, "a proposal alpha had already rated was reopened by alpha anyway" + rows = (await db_session.execute(select(ProposalReview).where( + ProposalReview.agent_id == "alpha" + ))).scalars().all() + assert [r.rating for r in rows] == [3], ( + f"the swallowed reopen mutated alpha's decision: {[r.rating for r in rows]}" + ) + + control = await client.post( + f"/agent/beta/proposals/{proposal.id}/reopen", + data={"guidance": "The other lab still gets a say."}, + headers=_auth(lab.pi_b_id), + ) + assert control.status_code == 302 + db_session.expire_all() + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 1, ( + "beta's reopen created nothing either, so the assertion above proves nothing" + ) + + +async def test_reopen_is_idempotent_under_a_replayed_post( + client, db_session, lab, proposal, slack_off, +): + """A stale page or the Back button replays the reopen POST. The guard must make the + second one a no-op rather than mint a duplicate channel. + + Control: the first POST is asserted to have created exactly one channel, so "still + one channel" is not satisfied by a reopen that never worked. + """ + for _ in range(2): + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": "Same guidance, submitted twice."}, + headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 302 + db_session.expire_all() + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 1 + + assert (await db_session.scalar( + select(func.count(ProposalReview.id)).where( + ProposalReview.thread_decision_id == proposal.id + ) + )) == 1, "the replayed reopen filed a second ProposalReview" + + +async def test_reopen_drives_the_slack_client_when_slack_is_on( + client, db_session, lab, proposal, slack_on, +): + """The Slack-on branch of the same wiring, with a recording fake standing in for + AgentSlackClient (the live workspace belongs to another agent). + + Asserts the migration really calls Slack — creates a private channel, invites the + other bot, posts the handover — and that the DB rows still land in the same + request. `no_outbound_side_effects` guarantees nothing reached slack_sdk. + """ + guidance = "Push on the kinetics readout, not the chemistry." + r = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": guidance}, headers=_auth(lab.pi_a_id), + ) + assert r.status_code == 302, r.text[:400] + + assert [c.agent_id for c in slack_on] == ["alpha", "beta"], ( + f"the migration did not build a client for each bot: {slack_on}" + ) + creator = slack_on[0] + assert creator.created_channels and creator.created_channels[0]["is_private"], ( + "no private channel was requested from Slack" + ) + new_name = creator.created_channels[0]["name"] + assert any("U_beta" in inv["users"] for inv in creator.invites), ( + f"the other bot was never invited to the new channel: {creator.invites}" + ) + posted_here = [p for p in creator.posted if p["channel"] == f"G_{new_name}"] + assert any(guidance in p["text"] for p in posted_here), ( + f"the guidance was never posted into the private channel: {posted_here}" + ) + origin_posts = [p for p in creator.posted if p["channel"] == f"C_{proposal.channel}"] + assert origin_posts and all(guidance not in p["text"] for p in origin_posts), ( + "the origin thread got no close marker, or it leaked the PI's guidance" + ) + assert all(p["thread_ts"] == proposal.thread_id for p in origin_posts), ( + "the close marker was posted top-level instead of in the origin thread" + ) + + db_session.expire_all() + ch = (await db_session.execute(select(AgentChannel).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))).scalar_one() + assert ch.channel_id == f"G_{new_name}" + review = (await db_session.execute(select(ProposalReview).where( + ProposalReview.thread_decision_id == proposal.id + ))).scalar_one() + assert review.rating == 0 + + +async def test_a_failed_migration_files_no_review( + client, db_session, lab, proposal, monkeypatch, +): + """If Slack refuses the channel, the reopen must leave NOTHING behind — no + half-written review that would make the proposal look acted-on and permanently + block the retry (the idempotency guard keys off any review by this agent). + + Positive control: the same request, with the fake repaired, writes both rows. + """ + refuse = {"on": True} + + async def _on(*args, **kwargs): + return True + + async def _token(db, agent_id): + return f"xoxb-fake-{agent_id}" + + class _Refusing(FakeSlackClient): + def create_private_channel(self, name): + if refuse["on"]: + return None + return super().create_private_channel(name) + + monkeypatch.setattr( + "src.services.private_channels._slack_enabled_for_migration", _on) + monkeypatch.setattr( + "src.services.private_channels._get_or_fail_bot_token", _token) + monkeypatch.setattr( + "src.services.private_channels._make_client", + lambda agent_id, bot_token: _Refusing(agent_id=agent_id, bot_token=bot_token), + ) + + bad = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": "This one will fail."}, headers=_auth(lab.pi_a_id), + ) + assert bad.status_code == 500, bad.status_code + db_session.expire_all() + assert (await db_session.scalar(select(func.count(ProposalReview.id)).where( + ProposalReview.thread_decision_id == proposal.id + ))) == 0, ( + "a failed migration still filed a ProposalReview — the idempotency guard will " + "now treat every retry as a duplicate and the proposal is stuck" + ) + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 0 + + refuse["on"] = False + good = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": "Retry after the outage."}, headers=_auth(lab.pi_a_id), + ) + assert good.status_code == 302, ( + f"the retry control also failed ({good.status_code}); the assertions above " + "cannot distinguish 'clean abort' from 'reopen never works'" + ) + db_session.expire_all() + assert (await db_session.scalar(select(func.count(ProposalReview.id)).where( + ProposalReview.thread_decision_id == proposal.id + ))) == 1 + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 1 + + +async def test_reopen_is_blocked_for_an_inactive_agent_but_rating_is_not( + client, db_session, lab, proposal, slack_off, +): + """The documented asymmetry in agent_page.py: an inactive agent's PI can still rate + a proposal (passive, DB-only) but cannot reopen it (re-injects the bot into a live + discussion). Both halves, so neither can silently flip. + """ + reg = (await db_session.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == "alpha") + )).scalar_one() + reg.status = "inactive" + await db_session.flush() + + blocked = await client.post( + f"/agent/alpha/proposals/{proposal.id}/reopen", + data={"guidance": "Please refine."}, headers=_auth(lab.pi_a_id), + ) + assert blocked.status_code == 403, ( + f"an inactive agent was reopened into a live discussion: {blocked.status_code}" + ) + db_session.expire_all() + assert (await db_session.scalar(select(func.count(AgentChannel.id)).where( + AgentChannel.visibility == VISIBILITY_COLLAB_PRIVATE + ))) == 0 + + allowed = await client.post( + f"/agent/alpha/proposals/{proposal.id}/review", + data={"rating": "2", "comment": "still allowed to rate"}, + headers=_auth(lab.pi_a_id), + ) + assert allowed.status_code == 302, ( + "rating was blocked too — the inactive state is not the narrow, " + f"reopen-only gate it is documented to be: {allowed.status_code}" + ) diff --git a/tests/integration/test_public_graph.py b/tests/integration/test_public_graph.py new file mode 100644 index 0000000..a02cbaf --- /dev/null +++ b/tests/integration/test_public_graph.py @@ -0,0 +1,763 @@ +"""Public routes: run-window arithmetic and the privacy boundary. + +``tests/characterization/test_public_routes.py`` already pins that the ten public +endpoints *render*. This file covers the two things rendering cannot show: + +1. **The arithmetic.** The graph routes slice one long-running simulation into + date-bounded **run windows** (``.notes/cohort-system-v2.md`` §1 renamed the + concept; the constants live in ``src/routers/public.py``). Three boundaries are + in play and each is tested *at the exact edge*, never at a comfortable value in + the middle: + + * ``decided_at >= window_start`` — inclusive lower edge + * ``decided_at < window_end`` — exclusive upper edge + * ``created_at >= window_start_bound``— inclusive lower edge, on *posts* + +2. **The privacy boundary.** No public route may render ``collab_private`` + content. Every absence assertion below is paired with a positive control in the + same test, because "nothing private leaked" is trivially true of a page that + renders nothing at all — which is exactly what these pages do when the seeding + is wrong. + +Real ASGI requests, real Postgres, real Jinja. Nothing is mocked. + +**The module-level graph cache is cleared before every request** (see ``_get``). +``_cached_graph_payload`` memoizes per parameter set for 60s across *all* callers, +so without this a test would read the previous test's payload. The cache itself is +therefore deliberately not exercised here. +""" + +import itertools +import json +import re +import uuid +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import text + +from src.routers import public as public_mod +from src.routers.public import ( + CABO_WINDOW_START, + JUNE_POST_START, + SCHULTZ_GROUP_END, + SCHULTZ_GROUP_START, + SCHULTZ_PILOT_END, + SCHULTZ_PILOT_ORCIDS, + SCHULTZ_PILOT_START, +) +from tests import factories + +pytestmark = pytest.mark.integration + +TICK = timedelta(microseconds=1) # the smallest interval Postgres timestamptz stores + +# The Cabo window is inlined in the /cabo-graph route rather than exported as a +# constant, so it is re-stated here and anchored to the label the page prints +# (see test_cabo_window_start_is_inclusive_and_its_end_is_exclusive). If someone +# moves the route's window without moving this, the label assertion fails. +CABO_START = datetime(2026, 4, 27, tzinfo=UTC) +CABO_END = datetime(2026, 5, 8, tzinfo=UTC) # exclusive +CABO_LABEL = "April 27 – May 7, 2026" + +# Distinctive, JSON-safe markers. Plain [A-Za-z0-9-] so Jinja's |tojson cannot +# escape them into something a substring search would miss. +PUBLIC_SUMMARY = "PUBLICPROPOSAL-9f3a2b" +PRIVATE_SUMMARY = "PRIVATEPROPOSAL-7c1d84" +PUBLIC_CONTENT = "PUBLICPOSTBODY-51ee0a" +PRIVATE_CONTENT = "PRIVATEPOSTBODY-2a77bc" + +# One representative in-window instant per graph route. /scripps-graph has no +# upper bound (decided_at >= March 1, forever), so it shares the Cabo instant. +ROUTE_WINDOWS = [ + ("/cabo-graph", datetime(2026, 5, 1, tzinfo=UTC)), + ("/scripps-graph", datetime(2026, 5, 1, tzinfo=UTC)), + ("/schultz-alumni-pilot", datetime(2026, 6, 2, tzinfo=UTC)), + ("/schultz-group-alumni", datetime(2026, 6, 7, tzinfo=UTC)), +] +GRAPH_ROUTES = [p for p, _ in ROUTE_WINDOWS] + +# The complete public surface, pinned. The route-inventory test below compares +# this against the live router, so an endpoint added to public.py without being +# classified here fails the suite rather than quietly escaping the privacy sweep. +ALL_PUBLIC_ROUTES = [ + ("GET", "/"), + ("POST", "/waitlist"), + ("GET", "/access-pending"), + ("POST", "/access-pending/email"), + ("GET", "/cabo-graph"), + ("GET", "/scripps-graph"), + ("GET", "/schultz-alumni-pilot"), + ("GET", "/schultz-group-alumni"), + ("POST", "/api/proposal-vote"), + ("POST", "/api/proposal-vote/{vote_id}/details"), +] + +# Three real Schultz-pilot ORCIDs paired with agent_ids that are also in the +# hardcoded Scripps bucket, so one roster satisfies all four routes' very +# different node-selection rules (ORCID list / Scripps set / all agents). +ROSTER = ( + ("su", "SuBot", "0000-0002-9859-4104"), # Andrew Su + ("lairson", "LairsonBot", "0000-0001-6701-996X"), # Luke Lairson + ("young", "YoungBot", "0000-0001-8562-5736"), # Travis Young +) + +_seq = itertools.count(770000) +_GRAPH_DATA_RE = re.compile(r'', re.S) + + +# --- harness --------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _clear_graph_cache(): + public_mod._GRAPH_CACHE.clear() + yield + public_mod._GRAPH_CACHE.clear() + + +async def _get(client, path, **kwargs): + """GET with the 60s payload cache dropped, so the response reflects this test.""" + public_mod._GRAPH_CACHE.clear() + return await client.get(path, **kwargs) + + +def _ip_headers() -> dict: + """A unique client IP per call, so the module-global rate limiters in + public.py (30 votes/min, 10 waitlist signups/hour, per IP, per process) cannot + make one test's result depend on how many tests ran before it.""" + return {"X-Real-IP": f"198.51.100.{next(_seq) % 254 + 1}"} + + +def _payload(response) -> dict: + """The graph JSON the template hands to D3, parsed out of the page.""" + m = _GRAPH_DATA_RE.search(response.text) + assert m, "the page rendered no #graph-data block at all" + return json.loads(m.group(1)) + + +def _pairs(response) -> set[frozenset]: + return {frozenset((link["source"], link["target"])) for link in _payload(response)["links"]} + + +def test_the_payload_extractor_distinguishes_populated_from_empty(): + """Control for the helper every assertion below leans on. An extractor that + returned ``{}`` on every page would make the link assertions vacuous.""" + + class _R: + text = ( + '" + ) + + assert _payload(_R())["nodes"] == [{"id": "su"}] + assert _pairs(_R()) == {frozenset(("su", "lairson"))} + + +@pytest.fixture +async def roster(db_session): + """Three agents visible to all four routes at once.""" + for _aid, _bot, orcid in ROSTER: + assert orcid in SCHULTZ_PILOT_ORCIDS, ( + f"{orcid} left SCHULTZ_PILOT_ORCIDS — /schultz-alumni-pilot would render " + "an empty graph and every assertion in this file would pass vacuously" + ) + for aid, _bot, _orcid in ROSTER: + assert aid in public_mod._SCRIPPS, ( + f"agent {aid} left the _SCRIPPS bucket — /scripps-graph would render empty" + ) + + out = {} + for aid, bot, orcid in ROSTER: + user = await factories.make_user( + db_session, orcid=orcid, email=f"{aid}@example.org", + institution="Scripps Research", + ) + out[aid] = await factories.make_agent( + db_session, user=user, agent_id=aid, bot_name=bot, + pi_name=f"PI {aid}", status="active", + ) + await db_session.flush() + return out + + +@pytest.fixture +async def run(db_session): + return await factories.make_simulation_run(db_session) + + +async def _seed_edge( + db, + run, + *, + a, + b, + decided_at, + summary, + visibility="public", + post_created_at=None, + content=None, +): + """One graph edge, as the engine actually writes it. + + An edge needs BOTH halves: a ``new_post`` ``AgentMessage`` whose ``created_at`` + clears ``window_start_bound``, and a ``proposal`` ``ThreadDecision`` on that + post's ``message_ts`` whose ``decided_at`` falls in the decision window. + ``post_created_at`` defaults to ``decided_at`` so the post boundary is never + accidentally the thing under test. + """ + ts = f"{next(_seq)}.000100" + await factories.make_agent_message( + db, run=run, agent_id=a, phase="new_post", message_ts=ts, + created_at=post_created_at if post_created_at is not None else decided_at, + visibility=visibility, + content=content if content is not None else f"body for {summary}", + sender_name=f"{a}Bot", posted_at=float(next(_seq)), + ) + decision = await factories.make_thread_decision( + db, run=run, thread_id=ts, agent_a=a, agent_b=b, outcome="proposal", + origin_visibility=visibility, decided_at=decided_at, summary_text=summary, + ) + await db.flush() + return decision + + +# --- run-window arithmetic: the exact edges -------------------------------- + + +def test_the_two_june_windows_are_adjacent(): + """The premise the boundary test rests on. If the windows ever stop touching, + 'lands in exactly one window' is no longer the property being tested — there + would be a gap (or an overlap) and the next test would be measuring something + else.""" + assert SCHULTZ_PILOT_END == SCHULTZ_GROUP_START, ( + f"pilot ends {SCHULTZ_PILOT_END}, group starts {SCHULTZ_GROUP_START}" + ) + assert SCHULTZ_PILOT_START < SCHULTZ_PILOT_END < SCHULTZ_GROUP_END + + +async def test_a_decision_exactly_on_the_shared_boundary_lands_in_exactly_one_window( + client, db_session, roster, run +): + """The instant Jun 5 00:00:00.000000Z belongs to the pilot window's exclusive + end AND the group window's inclusive start. It must be counted once. + + Catches: ``decided_at < :window_end`` weakened to ``<=`` (the edge would show + in both), and ``decided_at >= :decided_floor`` tightened to ``>`` (it would + show in neither). Both mutations survive any test that samples the middle of a + window. + + Control: the microsecond *before* the boundary lands in the pilot window and + only there, so 'exactly one' is not satisfied by a pair of broken pages. + """ + on = SCHULTZ_GROUP_START + await _seed_edge(db_session, run, a="su", b="lairson", decided_at=on, + summary="EDGE-ON-BOUNDARY") + await _seed_edge(db_session, run, a="su", b="young", decided_at=on - TICK, + summary="EDGE-ONE-TICK-EARLIER") + + pilot = await _get(client, "/schultz-alumni-pilot") + group = await _get(client, "/schultz-group-alumni") + assert pilot.status_code == 200 and group.status_code == 200 + + landed_in = [w for w, r in (("pilot", pilot), ("group", group)) + if "EDGE-ON-BOUNDARY" in r.text] + assert len(landed_in) == 1, ( + f"a decision at exactly {on.isoformat()} appeared in {landed_in or 'NO'} " + "window(s); the boundary instant must be counted exactly once" + ) + assert landed_in == ["group"], ( + "the boundary instant belongs to the window that OPENS on it " + "(>= start), not to the one that closes on it (< end)" + ) + + assert "EDGE-ONE-TICK-EARLIER" in pilot.text, ( + "control leg failed: the microsecond before the boundary is not in the " + "pilot window either, so the page is simply empty and the assertion " + "above proves nothing" + ) + assert "EDGE-ONE-TICK-EARLIER" not in group.text + + # And structurally, not just as text: each window holds its own single edge. + assert _pairs(pilot) == {frozenset(("su", "young"))} + assert _pairs(group) == {frozenset(("su", "lairson"))} + + +async def test_cabo_window_start_is_inclusive_and_its_end_is_exclusive( + client, db_session, roster, run +): + """Both edges of a closed-open window, at the exact instant, plus proof that + the excluded row is otherwise perfectly renderable. + + Catches: ``>=``→``>`` on the start (April 27 00:00 would vanish), ``<``→``<=`` + on the end (May 8 00:00 would appear), and a silent move of either constant + (the page's own label is asserted). + """ + await _seed_edge(db_session, run, a="su", b="lairson", decided_at=CABO_START, + summary="CABO-AT-START") + await _seed_edge(db_session, run, a="su", b="young", decided_at=CABO_END - TICK, + summary="CABO-LAST-INSTANT") + await _seed_edge(db_session, run, a="lairson", b="young", decided_at=CABO_END, + summary="CABO-AT-END") + + r = await _get(client, "/cabo-graph") + assert r.status_code == 200 + assert CABO_LABEL in r.text, ( + "the /cabo-graph route no longer claims the window this test asserts on; " + f"expected the page to say {CABO_LABEL!r}" + ) + assert "CABO-AT-START" in r.text, "the inclusive start instant was dropped" + assert "CABO-LAST-INSTANT" in r.text, "the last representable instant was dropped" + assert "CABO-AT-END" not in r.text, ( + f"a decision at exactly {CABO_END.isoformat()} appeared; window_end is exclusive" + ) + + # Control: the excluded row is a normal, public, well-formed proposal — it + # renders on /scripps-graph, whose window has no upper bound. So "CABO-AT-END + # is absent" is about the window arithmetic, not about a malformed fixture. + scripps = await _get(client, "/scripps-graph") + assert "CABO-AT-END" in scripps.text, ( + "control leg failed: the excluded decision does not render anywhere, so " + "its absence from /cabo-graph proves nothing about window_end" + ) + + +async def test_the_post_creation_bound_is_inclusive_at_the_exact_instant( + client, db_session, roster, run +): + """``window_posts`` bounds *posts* by ``created_at >= window_start_bound``, + independently of when the proposal was decided. Both decisions below sit well + inside the pilot decision window; only the originating posts differ, by one + microsecond across the bound. + + Catches: ``>=``→``>`` on the post bound, and removal of the + ``thread_id IN (SELECT message_ts FROM window_posts)`` join (which would let + the pre-bound post's edge through). + + Note the consequence, which is real rather than hypothetical: for + /schultz-alumni-pilot ``JUNE_POST_START == SCHULTZ_PILOT_START``, so that + window has ZERO lead-in — a thread opened May 31 and decided June 2 is + silently dropped. The route's own docstring warns against exactly this ("a + thread can be opened a couple days before its proposal lands"). + """ + decided = SCHULTZ_PILOT_START + timedelta(days=1) + await _seed_edge(db_session, run, a="su", b="lairson", decided_at=decided, + post_created_at=JUNE_POST_START, summary="POST-AT-BOUND") + await _seed_edge(db_session, run, a="su", b="young", decided_at=decided, + post_created_at=JUNE_POST_START - TICK, + summary="POST-ONE-TICK-BEFORE-BOUND") + + r = await _get(client, "/schultz-alumni-pilot") + assert r.status_code == 200 + assert "POST-AT-BOUND" in r.text, ( + "a post created at exactly window_start_bound was excluded; the bound is >=" + ) + assert "POST-ONE-TICK-BEFORE-BOUND" not in r.text, ( + "a post created one microsecond before window_start_bound was included" + ) + assert _pairs(r) == {frozenset(("su", "lairson"))} + + +async def test_a_decision_before_the_window_start_is_excluded( + client, db_session, roster, run +): + """The lower decision edge from the other side, with its control one tick later. + + Catches ``decided_at >= :decided_floor`` loosened to ``>=`` on a different + column, or dropped entirely — either of which would leak the whole simulation's + history into a window that claims four days of it. + """ + await _seed_edge(db_session, run, a="su", b="lairson", + decided_at=SCHULTZ_PILOT_START - TICK, summary="BEFORE-WINDOW") + await _seed_edge(db_session, run, a="su", b="young", + decided_at=SCHULTZ_PILOT_START, summary="AT-WINDOW-START") + + r = await _get(client, "/schultz-alumni-pilot") + assert r.status_code == 200 + assert "AT-WINDOW-START" in r.text, "the inclusive start instant was dropped" + assert "BEFORE-WINDOW" not in r.text, ( + "a decision one microsecond before window_start was included" + ) + + +async def test_only_the_first_proposal_on_a_thread_is_published( + client, db_session, roster, run +): + """``thread_first`` keeps the EARLIEST decision per (pair, thread). + + The query's own comment says why: a later row on the same thread is a + re-proposal made after a PI reopened and refined it, so its summary carries + human feedback that was never meant for the public graph. This is arithmetic + with a privacy edge, and nothing else asserts it. + + Catches: ``ORDER BY a, b, thread_id, decided_at ASC`` flipped to ``DESC`` (the + human-influenced text would be published), and removal of the ``DISTINCT ON`` + (the pair would be double-counted as two joint proposals). + + Control: the first proposal IS published, so "the later one is absent" is not + the empty page again. + """ + decided = datetime(2026, 6, 7, tzinfo=UTC) + ts = f"{next(_seq)}.000100" + await factories.make_agent_message( + db_session, run=run, agent_id="su", phase="new_post", message_ts=ts, + created_at=decided, visibility="public", content="the originating post", + posted_at=float(next(_seq)), + ) + for offset, summary in ((0, "FIRST-BOT-PROPOSAL"), (2, "LATER-REPROPOSAL-AFTER-PI")): + await factories.make_thread_decision( + db_session, run=run, thread_id=ts, agent_a="su", agent_b="lairson", + outcome="proposal", origin_visibility="public", + decided_at=decided + timedelta(hours=offset), summary_text=summary, + ) + await db_session.flush() + + r = await _get(client, "/schultz-group-alumni") + assert r.status_code == 200 + # Control first, so neither assertion below can pass on an empty page: the pair + # renders, exactly once, no matter which of the two summaries got picked. + links = _payload(r)["links"] + assert len(links) == 1 and links[0]["weight"] == 1, ( + f"one thread must render as one edge counting one joint proposal: {links}" + ) + assert "LATER-REPROPOSAL-AFTER-PI" not in r.text, ( + "the re-proposal made after a PI reopened and refined the thread was " + "published; only the bots' own first proposal belongs on the public graph" + ) + assert "FIRST-BOT-PROPOSAL" in r.text, ( + "the bots' first proposal was dropped along with the re-proposal" + ) + + +# --- an empty window must render ------------------------------------------ + + +@pytest.mark.parametrize("path,decided_at", ROUTE_WINDOWS, ids=[p for p, _ in ROUTE_WINDOWS]) +async def test_an_empty_window_renders_and_a_populated_one_shows_its_edge( + client, db_session, roster, run, path, decided_at +): + """An empty run window is a 200 with an empty graph, not a 500 — a page that + divides by ``len(nodes)`` or indexes ``palette[0]`` fails here. + + Control, in the same test: the same route with one in-window edge renders that + edge. Without it, "renders" is satisfied by a page that always shows nothing, + which is also what every privacy assertion in this file would then be testing. + """ + empty = await _get(client, path) + assert empty.status_code == 200, f"{path} 500s on an empty window" + assert "text/html" in empty.headers["content-type"] + payload = _payload(empty) + assert payload == {"nodes": [], "links": []}, f"{path} was not actually empty: {payload}" + + await _seed_edge(db_session, run, a="su", b="lairson", decided_at=decided_at, + summary=PUBLIC_SUMMARY) + populated = await _get(client, path) + assert populated.status_code == 200 + assert PUBLIC_SUMMARY in populated.text, ( + f"control leg failed: {path} shows nothing even with an in-window edge, so " + "'the empty window renders' above is not distinguishable from a broken route" + ) + assert _pairs(populated) == {frozenset(("su", "lairson"))} + + +# --- the privacy boundary -------------------------------------------------- + + +@pytest.mark.parametrize("path,decided_at", ROUTE_WINDOWS, ids=[p for p, _ in ROUTE_WINDOWS]) +async def test_no_graph_route_exposes_collab_private_content( + client, db_session, roster, run, path, decided_at +): + """THE load-bearing test: ``collab_private`` never reaches a public page. + + Seeds two real proposals in the same run and the same window — one ``public``, + one ``collab_private`` — and asserts the private one leaks nothing: not its + summary, not the body of the post it came from, not its decision id, and not + even the *edge*, whose bare existence would disclose that two named PIs are + collaborating privately. + + Catches: deleting ``AND origin_visibility = 'public'`` from the ``pairs`` CTE. + + Control, same test: the public proposal in the same window IS rendered. This + is the whole point — these pages show nothing at all under a dozen unrelated + faults, and "no private content" is true of every one of them. + """ + await _seed_edge(db_session, run, a="su", b="lairson", decided_at=decided_at, + summary=PUBLIC_SUMMARY, content=PUBLIC_CONTENT, visibility="public") + private = await _seed_edge( + db_session, run, a="su", b="young", decided_at=decided_at, + summary=PRIVATE_SUMMARY, content=PRIVATE_CONTENT, visibility="collab_private", + ) + + r = await _get(client, path) + assert r.status_code == 200 + + assert PUBLIC_SUMMARY in r.text, ( + f"control leg failed: {path} does not render the PUBLIC proposal either, so " + "the private-content assertions below are vacuous" + ) + assert frozenset(("su", "lairson")) in _pairs(r), "control leg failed: no public edge" + + assert PRIVATE_SUMMARY not in r.text, f"{path} LEAKED a collab_private summary" + assert PRIVATE_CONTENT not in r.text, f"{path} LEAKED a collab_private message body" + assert str(private.id) not in r.text, f"{path} LEAKED a collab_private decision id" + assert private.thread_id not in r.text, f"{path} LEAKED a collab_private thread id" + assert frozenset(("su", "young")) not in _pairs(r), ( + f"{path} disclosed the EXISTENCE of a private collaboration as a graph edge" + ) + assert "young" not in {n["id"] for n in _payload(r)["nodes"]}, ( + f"{path} rendered a PI whose only activity is private" + ) + + +async def _seed_every_window(db, run): + """A public + a private proposal inside every one of the four route windows.""" + seeded = [] + for _path, decided_at in ROUTE_WINDOWS: + await _seed_edge(db, run, a="su", b="lairson", decided_at=decided_at, + summary=PUBLIC_SUMMARY, content=PUBLIC_CONTENT) + seeded.append( + await _seed_edge(db, run, a="su", b="young", decided_at=decided_at, + summary=PRIVATE_SUMMARY, content=PRIVATE_CONTENT, + visibility="collab_private") + ) + return seeded + + +async def _exercise(client, method, path, ctx): + """Drive one public endpoint with a request it will actually accept.""" + if path == "/waitlist": + return await client.post(path, data={"email": "sweep@example.edu"}, + headers=_ip_headers()) + if path == "/access-pending/email": + return await client.post(path, data={"email": "sweep@example.edu"}, + headers=_ip_headers()) + if path == "/api/proposal-vote": + # Aimed straight at the private decision: the endpoint must neither accept + # the vote nor echo anything about the row back. + return await client.post( + path, + json={"decision_id": str(ctx["private_id"]), "vote": "up", + "voter_token": "sweep-tok"}, + headers=_ip_headers(), + ) + if path == "/api/proposal-vote/{vote_id}/details": + return await client.post( + f"/api/proposal-vote/{ctx['vote_id']}/details", + json={"details": "sweep", "voter_token": "sweep-tok"}, + headers=_ip_headers(), + ) + assert method == "GET", f"no request builder for {method} {path}" + return await _get(client, path, headers=_ip_headers()) + + +@pytest.mark.parametrize( + "method,path", ALL_PUBLIC_ROUTES, ids=[f"{m} {p}" for m, p in ALL_PUBLIC_ROUTES] +) +async def test_every_public_route_withholds_collab_private_content( + client, db_session, roster, run, method, path +): + """The same private proposal, held against all ten public endpoints. + + The four graph routes carry a positive control (the public proposal renders). + The other six render no message content at all by design, so their control is + different in kind but not weaker: the test asserts, by direct query in the same + transaction, that the private row WAS present and visible while the request ran. + Without that leg a fixture that silently failed to seed would score green here. + """ + private_rows = await _seed_every_window(db_session, run) + public_decision = (await db_session.execute( + text( + "SELECT id FROM thread_decisions " + "WHERE summary_text = :s AND origin_visibility = 'public' LIMIT 1" + ), + {"s": PUBLIC_SUMMARY}, + )).scalar_one() + created = await client.post( + "/api/proposal-vote", + json={"decision_id": str(public_decision), "vote": "up", "voter_token": "sweep-tok"}, + headers=_ip_headers(), + ) + assert created.status_code == 200, f"could not seed a vote to exercise: {created.text}" + ctx = {"private_id": private_rows[0].id, "vote_id": created.json()["id"]} + + # Control for every case: the private content really is in the database, in + # this transaction, right now. + stored = (await db_session.execute( + text( + "SELECT count(*) FROM thread_decisions " + "WHERE summary_text = :s AND origin_visibility = 'collab_private'" + ), + {"s": PRIVATE_SUMMARY}, + )).scalar_one() + assert stored == len(ROUTE_WINDOWS), ( + f"control leg failed: expected {len(ROUTE_WINDOWS)} private proposals in the " + f"DB, found {stored}; an absence assertion against no data proves nothing" + ) + + r = await _exercise(client, method, path, ctx) + assert r.status_code < 500, f"{method} {path} -> {r.status_code}: {r.text[:400]}" + + body = r.text + for marker, what in ( + (PRIVATE_SUMMARY, "a collab_private proposal summary"), + (PRIVATE_CONTENT, "a collab_private message body"), + ): + assert marker not in body, f"{method} {path} LEAKED {what}" + for row in private_rows: + assert str(row.id) not in body, f"{method} {path} LEAKED a private decision id" + assert row.thread_id not in body, f"{method} {path} LEAKED a private thread id" + + if path in GRAPH_ROUTES: + assert PUBLIC_SUMMARY in body, ( + f"control leg failed: {path} rendered no public proposal either" + ) + + if path == "/api/proposal-vote": + # This endpoint renders no content, so "the marker is absent" is true of it + # no matter what. What it CAN leak is existence: a 200 tells an anonymous + # caller that the private proposal id is real. Only a 404 withholds that. + assert r.status_code == 404, ( + f"the vote endpoint answered {r.status_code} for a collab_private " + "proposal; anything but 404 confirms the private row exists" + ) + + +def test_the_privacy_sweep_covers_every_public_route(): + """A new endpoint in public.py must be visibly absent, not silently uncovered. + + ``ALL_PUBLIC_ROUTES`` is compared against the live router, so adding a route — + say, a page that lists recent proposals — fails here until it is classified and + swept. ``GRAPH_ROUTES`` (the content-rendering subset) is asserted to be a real, + non-empty subset, so the classification cannot be emptied to make this pass. + """ + live = { + (m, r.path) + for r in public_mod.router.routes + for m in getattr(r, "methods", set()) + if m in {"GET", "POST", "PUT", "PATCH", "DELETE"} + } + pinned = set(ALL_PUBLIC_ROUTES) + assert live == pinned, ( + "src/routers/public.py's route surface changed.\n" + f" new / unswept: {sorted(live - pinned)}\n" + f" gone: {sorted(pinned - live)}\n" + "Classify each new route: add it to ALL_PUBLIC_ROUTES, and to ROUTE_WINDOWS " + "too if it renders message content." + ) + assert {("GET", p) for p in GRAPH_ROUTES} <= pinned, ( + "a content-rendering route in GRAPH_ROUTES is not a real public GET route" + ) + assert len(GRAPH_ROUTES) == 4, ( + "the content-rendering classification was emptied or expanded without " + "updating this pin" + ) + + +async def test_the_vote_endpoint_refuses_a_private_proposal_but_accepts_a_public_one( + client, db_session, run +): + """Both legs in one test. The characterization suite pins each half separately; + a 404-for-everything endpoint satisfies the private half on its own, so the two + are asserted together here. + + Catches: deleting ``AND origin_visibility = 'public'`` from the vote lookup, + which would both write rows for private proposals and confirm their existence. + """ + private = await factories.make_thread_decision( + db_session, run=run, outcome="proposal", origin_visibility="collab_private", + summary_text=PRIVATE_SUMMARY, + ) + public = await factories.make_thread_decision( + db_session, run=run, outcome="proposal", origin_visibility="public", + summary_text=PUBLIC_SUMMARY, + ) + await db_session.flush() + + refused = await client.post( + "/api/proposal-vote", + json={"decision_id": str(private.id), "vote": "up", "voter_token": "vote-tok"}, + headers=_ip_headers(), + ) + assert refused.status_code == 404, ( + f"a collab_private proposal was votable: {refused.status_code} {refused.text[:300]}" + ) + assert PRIVATE_SUMMARY not in refused.text + assert str(private.id) not in refused.text, ( + "the 404 body echoed the private decision id back" + ) + + accepted = await client.post( + "/api/proposal-vote", + json={"decision_id": str(public.id), "vote": "up", "voter_token": "vote-tok"}, + headers=_ip_headers(), + ) + assert accepted.status_code == 200, ( + "control leg failed: the endpoint refuses public proposals too, so the 404 " + f"above says nothing about visibility ({accepted.status_code} {accepted.text[:300]})" + ) + assert uuid.UUID(accepted.json()["id"]) + + # And nothing was written for the private one. + votes = (await db_session.execute( + text("SELECT count(*) FROM proposal_votes WHERE thread_decision_id = :d"), + {"d": str(private.id)}, + )).scalar_one() + assert votes == 0, "a vote row was persisted against a collab_private proposal" + + +async def test_a_private_decision_on_a_public_post_still_does_not_render( + client, db_session, roster, run +): + """``window_posts`` does not filter on ``agent_messages.visibility`` — the + only privacy filter in the edge query is ``origin_visibility`` on the decision. + This pins that the single filter is load-bearing on its own: a private decision + hanging off a perfectly public post is still withheld. + + Control: the public decision on an equally public post does render. + """ + decided = datetime(2026, 6, 7, tzinfo=UTC) + await _seed_edge(db_session, run, a="su", b="lairson", decided_at=decided, + summary=PUBLIC_SUMMARY, visibility="public") + + ts = f"{next(_seq)}.000100" + await factories.make_agent_message( + db_session, run=run, agent_id="su", phase="new_post", message_ts=ts, + created_at=decided, visibility="public", content="an ordinary public post", + posted_at=float(next(_seq)), + ) + await factories.make_thread_decision( + db_session, run=run, thread_id=ts, agent_a="su", agent_b="young", + outcome="proposal", origin_visibility="collab_private", decided_at=decided, + summary_text=PRIVATE_SUMMARY, + ) + await db_session.flush() + + r = await _get(client, "/schultz-group-alumni") + assert r.status_code == 200 + assert PUBLIC_SUMMARY in r.text, "control leg failed: no public edge rendered" + assert PRIVATE_SUMMARY not in r.text, ( + "a collab_private decision leaked because its originating post was public" + ) + assert _pairs(r) == {frozenset(("su", "lairson"))} + + +# --- window constants are internally consistent ---------------------------- + + +def test_the_declared_windows_do_not_overlap_or_invert(): + """Cheap arithmetic on the constants themselves. A typo that made a window + inverted (start > end) would render an always-empty page, and every 'renders' + test in the characterization suite would still pass.""" + for name, start, end in ( + ("cabo", CABO_START, CABO_END), + ("schultz pilot", SCHULTZ_PILOT_START, SCHULTZ_PILOT_END), + ("schultz group", SCHULTZ_GROUP_START, SCHULTZ_GROUP_END), + ): + assert start < end, f"{name} window is inverted: {start} .. {end}" + assert CABO_WINDOW_START <= CABO_START, ( + "the Cabo post bound must not sit after the decision window it feeds" + ) + assert JUNE_POST_START <= SCHULTZ_PILOT_START + assert JUNE_POST_START <= SCHULTZ_GROUP_START + assert CABO_END <= SCHULTZ_PILOT_START, "the Cabo and June windows overlap" diff --git a/tests/integration/test_role_live_flip.py b/tests/integration/test_role_live_flip.py new file mode 100644 index 0000000..d770629 --- /dev/null +++ b/tests/integration/test_role_live_flip.py @@ -0,0 +1,99 @@ +"""A `agents.role` change on a live agent is picked up without a process restart. + +`_sync_roster_from_db` (src/agent/simulation.py) polls `AgentRegistry` on a timer to add +newly-activated agents and remove deactivated ones. Before this test existed, a role +change on a *surviving* agent (still active, still present in the roster) was invisible: +the method computed `to_add`/`to_remove` and returned early whenever both were empty, +which is exactly the case for a pure role reassignment. This exercises that path with a +real Postgres: seed one active agent at role 'pi_lab', update its DB row to 'scout_hub' +in a separate committed session (mirroring how the admin UI would write it), force the +poll throttle open, and assert the in-memory `Agent.role` picks up the change on the next +sync tick. + +Not using the rolled-back `db_session` fixture on purpose: the engine opens its own +sessions via `session_factory` and the test needs its own UPDATE to be visible to those +sessions the way it would be in production (two independent connections, both committed), +not merely visible within one shared, uncommitted transaction. +""" + +import uuid + +import pytest +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.transport import NullTransport +from src.models import AgentRegistry, SimulationRun + +pytestmark = pytest.mark.integration + +AGENT_ID = "role-flip-test-agent" + + +@pytest.fixture +async def engine_with_one_agent(engine): + """A real `SimulationEngine` wired to the migrated test Postgres, with one + active `AgentRegistry` row seeded at role='pi_lab'. + + Slack is off (NullTransport) — this test is about the roster/role sync, not + transport. Yields `(sim_engine, session_factory, agent_id)`. + """ + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + db.add(AgentRegistry( + agent_id=AGENT_ID, + bot_name="RoleFlipTestBot", + pi_name="PI Role Flip", + status="active", + role="pi_lab", + )) + await db.commit() + + sim_engine = SimulationEngine( + agents=[Agent(agent_id=AGENT_ID, bot_name="RoleFlipTestBot", + pi_name="PI Role Flip", role="pi_lab")], + slack_clients={AGENT_ID: NullTransport(AGENT_ID)}, + budget_cap=0, + session_factory=factory, + simulation_run_id=run_id, + slack_enabled=False, + ) + + try: + yield sim_engine, factory, AGENT_ID + finally: + async with factory() as db: + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id == AGENT_ID)) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +async def test_role_change_is_picked_up_without_restart(engine_with_one_agent): + """A DB role change on a running agent updates Agent.role on the next sync.""" + engine, session_factory, agent_id = engine_with_one_agent + assert engine.agents[agent_id].role == "pi_lab" + + async with session_factory() as db: + await db.execute( + update(AgentRegistry).where(AgentRegistry.agent_id == agent_id).values(role="scout_hub") + ) + await db.commit() + + engine._last_roster_poll = 0.0 # force the throttle open + await engine._sync_roster_from_db() + + assert engine.agents[agent_id].role == "scout_hub" + + # Sanity check against the DB directly, so a false pass (e.g. the sync silently + # no-oping and the assertion above passing only because nothing ever changed + # `.role` back) can't hide behind the in-memory assertion alone. + async with session_factory() as db: + row = (await db.execute( + select(AgentRegistry).where(AgentRegistry.agent_id == agent_id) + )).scalar_one() + assert row.role == "scout_hub" diff --git a/tests/integration/test_slack_client_live.py b/tests/integration/test_slack_client_live.py new file mode 100644 index 0000000..cd9ab05 --- /dev/null +++ b/tests/integration/test_slack_client_live.py @@ -0,0 +1,532 @@ +"""Every AgentSlackClient method against the real workspace. + +Rule S1 throughout: each write is verified by reading Slack back, not by checking that +no exception escaped. Several of these methods return `None` or `[]` on failure, so +"it didn't raise" is not evidence of anything. + +One trap the offline contract tests surfaced first: `post_message` runs the text through +`markdown_to_mrkdwn`, so what lands in Slack is not byte-identical to what we passed. +Every live assertion here uses plain prose for that reason. +""" + +import os +import re +import time +import uuid + +import pytest + +from src.agent.slack_client import BotNotInvitedToPrivateChannel, ThreadNotFound + +pytestmark = [pytest.mark.integration, pytest.mark.live_slack] + +# Slack allows roughly one message per second per channel. Every test here posts a +# handful; this keeps a full-file run comfortably inside that. +POST_GAP = 1.1 + + +def _post(client, channel, text, thread_ts=None): + out = client.post_message(channel, text, thread_ts=thread_ts) + time.sleep(POST_GAP) + return out + + +# --- identity ---------------------------------------------------------------------- + + +def test_connect_and_identity(slack_client_su, slack_pi_user_id): + assert slack_client_su.is_connected is True + uid = slack_client_su.bot_user_id + assert uid and uid.startswith("U"), uid + assert slack_client_su.is_bot_user(uid) is True + # Control: a human is not a bot. Without it, an is_bot_user that returned True + # unconditionally would pass. + assert slack_client_su.is_bot_user(slack_pi_user_id) is False + + +def test_resolve_user_name_returns_a_name_not_the_raw_id(slack_client_su, slack_pi_user_id): + """A fallback to the raw id is what you get when users:read is missing, and it is + silent — the PI's messages would render as U0123ABC in every prompt.""" + name = slack_client_su.resolve_user_name(slack_pi_user_id) + assert name and name != slack_pi_user_id, f"fell back to the raw id: {name!r}" + + +def test_an_unknown_user_id_does_not_raise(slack_client_su): + """Degrade, don't crash: an unresolvable id must not take down a turn.""" + assert slack_client_su.resolve_user_name("U000NOTREAL") is not None + + +# --- channel lifecycle -------------------------------------------------------------- + + +def test_channel_create_list_join_and_id_resolution( + slack_client_su, slack_probe_channel, slack_list_all_channels +): + """Creation, resolution and join. The channel's *existence* is asserted against the + fully paginated fixture rather than against `list_channels()` — deliberately a + different code path, so this test cannot pass because the client's own listing and + the client's own resolution share a bug. `list_channels()`'s completeness is the + subject of test_list_channels_returns_every_public_channel below, and is not + re-litigated here. + """ + name, cid = slack_probe_channel + assert slack_list_all_channels(slack_client_su).get(name) == cid, ( + f"#{name} was created but Slack does not list it as a public channel" + ) + # list_channels itself must at least answer with a well-formed page. + listed = slack_client_su.list_channels() + assert listed and all(v.startswith("C") for v in listed.values()), listed + + # create_channel populates the name->id cache, which is what makes resolution work + # without a listing round trip. That is the contract `cache_channel_ids` and + # `_ensure_seeded_channels` rely on. + assert slack_client_su._channel_name_to_id.get(name) == cid, ( + "create_channel did not cache the new channel's id" + ) + assert slack_client_su.get_channel_id(name) == cid + assert slack_client_su._resolve_channel_id(name) == cid + assert slack_client_su._resolve_channel_id(cid) == cid, "an id must pass through" + # join is idempotent — the engine calls it on every post via autojoin. + slack_client_su.join_channel(cid) + slack_client_su.join_channel(cid) + # Control: an unknown name resolves to None rather than to something plausible. + assert slack_client_su.get_channel_id("t-does-not-exist-zzzz") is None + + +def test_list_channels_returns_every_public_channel( + slack_client_su, slack_list_all_channels +): + """Pagination, against the workspace that broke without it. + + This was the root cause of the whole tier's rotating failures: `list_channels` + asked conversations.list for a single 200-item page and ignored + `response_metadata.next_cursor`, so every test that addressed a channel by name + went through a listing that could silently omit it. Slack orders conversations.list + by channel id, and ids are not monotonic in creation time, so which channels a + single page showed was effectively random. + + The control matters as much as the claim: the workspace must be *bigger* than one + page, or a client that still ignored the cursor would pass this. + + The ground truth is read TWICE, bracketing the call under test, and the comparison is + made against both. Neither walk is a snapshot — conversations.list is cursor-paginated + over a workspace this very suite mutates (the previous test archives its probe channel + on the way out) and Slack's listing is eventually consistent, so a channel can be + absent from one complete walk and present in the next one seconds later. Measured: + with a single ground read, this test failed on 1 of 3 consecutive tier runs because a + `t-probe-` channel archived moments earlier was missing from the ground walk and + present in `list_channels()` — an artifact of Slack's index latency, asserted as if it + were a src defect. Bracketing keeps both real claims intact: a channel Slack listed in + both walks was there throughout and a paginating client must have seen it, and a name + the client invented is in neither. + """ + before = slack_list_all_channels(slack_client_su) + listed = slack_client_su.list_channels() + after = slack_list_all_channels(slack_client_su) + + stable = set(before) & set(after) + assert len(stable) > 200, ( + f"only {len(stable)} stably-listed public channels — this workspace no longer " + "exceeds one 200-item page, so this test can no longer detect a missing paginator" + ) + missing = sorted(stable - set(listed)) + assert not missing, ( + f"list_channels() returned {len(listed)} of {len(stable)} public channels that " + f"Slack listed in two independent walks; {len(missing)} are invisible to it, " + f"e.g. {missing[:5]}" + ) + invented = sorted(set(listed) - set(before) - set(after)) + assert not invented, ( + "list_channels() returned channels Slack does not list in either walk: " + f"{invented[:5]}" + ) + # The ids agree too, not just the names — a listing that paired the right names with + # the wrong ids resolves every post to the wrong channel. + assert {n: listed[n] for n in stable} == {n: before[n] for n in stable} + + +def test_exclude_archived_is_opt_in_because_an_archived_channel_owns_its_name( + slack_clients, slack_list_all_channels +): + """Both halves of the `exclude_archived` decision, live. + + The default is False — archived channels ARE listed — and that is deliberate, not + an oversight: both callers ask this question to learn whether a *name* is in use, + and Slack keeps the name of an archived channel reserved. A listing that hid + archived channels would send `_ensure_seeded_channels` to conversations.create for + a name Slack refuses with `name_taken`, which is the same production failure the + pagination fix just closed, reached by a different route. + + Control: passing True really does drop it, so the parameter is not inert. + """ + su = slack_clients["su"] + name = f"t-arch-{uuid.uuid4().hex[:8]}" + made = su.create_channel(name) + assert made and made.get("id"), made + su._call_with_retry(su._client.conversations_archive, channel=made["id"]) + + with_archived = su.list_channels() + assert with_archived.get(name) == made["id"], ( + f"#{name} is archived and vanished from the default listing — " + "_ensure_seeded_channels would try to create it and get name_taken" + ) + without = su.list_channels(exclude_archived=True) + assert name not in without, ( + "exclude_archived=True still returned an archived channel, so the flag does " + "nothing" + ) + assert without and set(without) < set(with_archived), ( + f"exclude_archived=True is not a subset of the default listing: " + f"{len(without)} vs {len(with_archived)}" + ) + # And the archived channel is still addressable by name, which is the point. + assert su.get_channel_id(name) == made["id"] + + +def test_cache_channel_ids_is_used_by_resolution(slack_client_su): + """The engine seeds this cache from the DB so it does not re-list on every post.""" + slack_client_su.cache_channel_ids({"t-cached-name": "C_CACHED_FAKE"}) + assert slack_client_su._resolve_channel_id("t-cached-name") == "C_CACHED_FAKE" + + +# --- posting, threading, history ---------------------------------------------------- + + +def test_post_thread_and_history_round_trip(slack_client_su, slack_probe_channel): + name, cid = slack_probe_channel + root = _post(slack_client_su, cid, "root from the probe") + assert root and root.get("ts"), root + reply = _post(slack_client_su, cid, "reply from the probe", thread_ts=root["ts"]) + assert reply and reply["ts"] != root["ts"] + + hist = slack_client_su.poll_channel_messages(cid, oldest="0") + texts = [m.get("text") for m in hist] + assert "root from the probe" in texts + # A threaded reply must NOT surface as a top-level history entry, or every reply + # would be re-ingested as a fresh root by the poller. + assert "reply from the probe" not in texts, ( + f"the reply appeared at top level — it was not threaded. history={texts}" + ) + + replies = slack_client_su.get_thread_replies(cid, root["ts"]) + assert "reply from the probe" in [m.get("text") for m in replies] + assert len(slack_client_su.get_full_channel_history(cid)) >= 1 + assert len(slack_client_su.get_all_thread_replies(cid, root["ts"])) >= 1 + + +def test_poll_cursor_excludes_already_seen_messages(slack_client_su, slack_probe_channel): + """The engine's _poll_cursors depends on this. A poll that ignored `oldest` would + re-ingest the whole channel every tick and duplicate every message.""" + name, cid = slack_probe_channel + first = _post(slack_client_su, cid, "before the cursor") + seen = slack_client_su.poll_channel_messages(cid, oldest="0") + assert "before the cursor" in [m.get("text") for m in seen] + + after = slack_client_su.poll_channel_messages(cid, oldest=first["ts"]) + assert "before the cursor" not in [m.get("text") for m in after], ( + "oldest= did not exclude the message at that ts" + ) + # Control: a NEW message past the cursor IS returned, so the filter is a cursor + # rather than a poll that returns nothing. + _post(slack_client_su, cid, "after the cursor") + after2 = slack_client_su.poll_channel_messages(cid, oldest=first["ts"]) + assert "after the cursor" in [m.get("text") for m in after2] + + +def test_markdown_is_rendered_as_slack_mrkdwn(slack_client_su, slack_probe_channel): + """Confirms live what the offline contract test asserts about the outbound call.""" + name, cid = slack_probe_channel + _post(slack_client_su, cid, "a **bold** claim") + texts = [m.get("text") for m in slack_client_su.poll_channel_messages(cid, oldest="0")] + assert "a *bold* claim" in texts, texts + + +def test_replying_to_a_nonexistent_thread_raises_thread_not_found( + slack_client_su, slack_probe_channel +): + name, cid = slack_probe_channel + with pytest.raises(ThreadNotFound): + slack_client_su.post_message(cid, "reply into the void", thread_ts="1111111111.000100") + + +def test_posting_to_a_nonexistent_channel_returns_none(slack_client_su): + """Degrade rather than crash: a stale channel id must not end a turn.""" + assert slack_client_su.post_message("C00000000000", "nowhere") is None + + +# --- the >4000-char split, at the client boundary -------------------------------------- + + +def _prose(n: int) -> str: + """Word-separated prose of exactly n characters.""" + unit = "kinetics " + s = (unit * (n // len(unit) + 2))[:n] + return s[:-1] + "." if s.endswith(" ") else s + + +def _texts_in(client, cid) -> list[str]: + """Every message in the channel, top level and threaded, oldest first.""" + out = [] + for msg in client.get_full_channel_history(cid): + out.append(msg.get("text") or "") + if msg.get("reply_count"): + for r in client.get_all_thread_replies(cid, msg["ts"]): + if r.get("ts") != msg.get("ts"): + out.append(r.get("text") or "") + return out + + +def test_a_message_at_the_limit_is_one_message(slack_client_su, slack_probe_channel): + """Measured live: Slack accepts exactly 4000 characters as a single message, so the + client must not split at the boundary and turn one post into two.""" + from src.agent.slack_client import SLACK_MAX_TEXT_CHARS + + name, cid = slack_probe_channel + body = _prose(SLACK_MAX_TEXT_CHARS) + assert len(body) == 4000 + out = _post(slack_client_su, cid, body) + assert out and len(out["posted_messages"]) == 1, out["posted_messages"] + assert len(_texts_in(slack_client_su, cid)) == 1 + + +@pytest.mark.parametrize("size", [4001, 8500]) +def test_an_over_limit_post_reports_every_message_it_created( + slack_client_su, slack_probe_channel, size +): + """Slack splits a >4000-char `text` itself and returns only the LAST chunk's ts, so + a client that posts blind names the tail of its own message and leaves the head with + no record. Chunking here instead means every Slack message is one we can account for. + + 4001 is the first size past the boundary; 8500 forces three chunks. Both are asserted + the same way, because the property — not the chunk count — is what matters: + `posted_messages` must be exactly the set of messages the channel now holds, in order, + and its FIRST ts (not its last) must be what `post_message` returns for threading. + """ + name, cid = slack_probe_channel + body = _prose(size) + out = _post(slack_client_su, cid, body) + assert out, "the oversized post did not land at all" + posted = out["posted_messages"] + assert len(posted) >= 2, f"{size} chars was not split: {len(posted)} message(s)" + assert out["ts"] == posted[0]["ts"], ( + "post_message returned a ts other than the first message's — this is the value " + "the engine records as the canonical id and threads replies onto" + ) + + live = _texts_in(slack_client_su, cid) + assert len(live) == len(posted), ( + f"Slack holds {len(live)} message(s) for {len(posted)} reported: {live[:2]}" + ) + # Every reported chunk is really there, and nothing else is. + from src.agent.slack_client import markdown_to_mrkdwn + assert [markdown_to_mrkdwn(p["text"]) for p in posted] == live + # No content was lost or duplicated across the split. + assert re.sub(r"\s+", "", "".join(live)) == re.sub(r"\s+", "", body) + # A split root stays ONE top-level post: the continuations hang off the first + # message, so nobody else's Phase 2 scan sees several roots for one post. + assert posted[0]["thread_ts"] is None + assert all(p["thread_ts"] == posted[0]["ts"] for p in posted[1:]), ( + f"continuation chunks are not threaded on the first: {[p['thread_ts'] for p in posted]}" + ) + assert len(slack_client_su.get_full_channel_history(cid)) == 1, ( + "the split produced more than one top-level message" + ) + + +def test_an_over_limit_reply_keeps_every_chunk_in_the_caller_s_thread( + slack_client_su, slack_probe_channel +): + """Control for the test above: for a *reply*, every chunk belongs to the thread the + caller named — not to a sub-thread on the first chunk.""" + name, cid = slack_probe_channel + root = _post(slack_client_su, cid, "root for a long reply") + out = _post(slack_client_su, cid, _prose(9000), thread_ts=root["ts"]) + posted = out["posted_messages"] + assert len(posted) >= 3, len(posted) + assert all(p["thread_ts"] == root["ts"] for p in posted), ( + f"a reply chunk left the thread: {[p['thread_ts'] for p in posted]}" + ) + replies = slack_client_su.get_all_thread_replies(cid, root["ts"]) + assert len([r for r in replies if r["ts"] != root["ts"]]) == len(posted) + + +def test_a_code_fence_spanning_a_split_is_closed_and_reopened( + slack_client_su, slack_probe_channel +): + """Slack renders `text` as mrkdwn, so a chunk that ends inside a ``` block renders + its tail as code and the next chunk renders its head as prose — the split moves the + block boundary. Balancing each chunk keeps every piece rendering as the whole would. + """ + name, cid = slack_probe_channel + body = "Here is the analysis script:\n\n```\n" + "\n".join( + f"row_{i} = measure(sample_{i}) # covalent engagement at t={i}" for i in range(120) + ) + "\n```\n\nThat is the whole pipeline." + assert len(body) > 4000, len(body) + out = _post(slack_client_su, cid, body) + posted = out["posted_messages"] + assert len(posted) >= 2, len(posted) + for i, p in enumerate(posted): + assert p["text"].count("```") % 2 == 0, ( + f"chunk {i} leaves a code fence open: ...{p['text'][-60:]!r}" + ) + live = _texts_in(slack_client_su, cid) + assert len(live) == len(posted) + # The fence repair is the only text added; every original line survives. + joined = "".join(live) + for i in (0, 60, 119): + assert f"row_{i} = measure(sample_{i})" in joined + + +# --- DMs ----------------------------------------------------------------------------- + + +def test_dm_send_lands_in_slack_but_is_not_polled_back(slack_client_su, slack_pi_user_id): + """`poll_dm_messages` filters to messages FROM the target user, excluding the bot's + own. That filter is load-bearing: `handle_dm` replies to whatever the poll returns, + so a bot that saw its own DM would answer itself forever. + + Both halves. The bot's message must really be in the DM channel (read back + unfiltered, Rule S1) and must be absent from the filtered poll. + """ + dm = slack_client_su.open_dm_channel(slack_pi_user_id) + assert dm and dm.startswith("D"), dm + marker = f"probe DM {uuid.uuid4().hex[:8]}" + sent = slack_client_su.send_dm(slack_pi_user_id, marker) + time.sleep(POST_GAP) + assert sent and sent.get("ts") + + raw = slack_client_su.poll_channel_messages(dm, oldest="0") + assert marker in [m.get("text") for m in raw], ( + "the DM never reached Slack at all" + ) + filtered = slack_client_su.poll_dm_messages(slack_pi_user_id, oldest="0") + assert marker not in [m.get("text") for m in filtered], ( + "poll_dm_messages returned the bot's own message — handle_dm would reply to " + "itself in a loop" + ) + assert all(m.get("user") == slack_pi_user_id for m in filtered), ( + f"poll_dm_messages returned a message from someone else: {filtered}" + ) + + +def test_open_dm_channel_is_cached(slack_client_su, slack_pi_user_id): + a = slack_client_su.open_dm_channel(slack_pi_user_id) + b = slack_client_su.open_dm_channel(slack_pi_user_id) + assert a == b and slack_client_su._dm_channels[slack_pi_user_id] == a + + +# --- private channels, and the groups:write finding ------------------------------------ + + +@pytest.fixture +def private_channel(slack_clients): + """A private channel created by su, archived on teardown.""" + su = slack_clients["su"] + requested = f"t-priv-{uuid.uuid4().hex[:8]}" + data = su.create_private_channel(requested) + assert data and data.get("id"), ( + f"could not create private #{requested}: {data} — if this is missing_scope, su " + "was installed without groups:write" + ) + # create_private_channel appends a UTC timestamp for collision avoidance, so the + # assigned name is not the requested one. Hand back what Slack actually made. + assert data["name"].startswith(requested), data["name"] + yield data["name"], data["id"] + try: + su._call_with_retry(su._client.conversations_archive, channel=data["id"]) + except Exception as exc: + print(f"WARNING: could not archive #{data['name']}: {exc}") + + +def test_private_channel_creation_needs_groups_write(slack_clients): + """The live A/B behind the BOT_SCOPES finding. + + wiseman was installed with exactly the BOT_SCOPES list as it shipped before this + work; su was installed with groups:write added. Same code, same call, different + grant — so a failure here is the scope and nothing else. + + This is why the fix is a scope-coverage test rather than a one-line edit: a bot + provisioned from the old manifest connects, posts and polls perfectly, and only + fails the one call that PI pairing depends on. + """ + su, wiseman = slack_clients["su"], slack_clients["wiseman"] + + ok = su.create_private_channel(f"t-priv-ab-{uuid.uuid4().hex[:6]}") + assert ok and ok.get("id"), f"su (with groups:write) could not create: {ok}" + try: + bad = wiseman.create_private_channel(f"t-priv-ab-{uuid.uuid4().hex[:6]}") + assert bad is None or not bad.get("id"), ( + "wiseman has no groups:write yet created a private channel — the A/B is " + f"broken, re-check the install scopes. got {bad}" + ) + finally: + su._call_with_retry(su._client.conversations_archive, channel=ok["id"]) + + +def test_private_channel_invite_and_membership(slack_clients, private_channel): + """A bot cannot self-join a private channel — it must be invited. That distinction + is exactly what _is_private_channel exists to protect.""" + name, cid = private_channel + su, cravatt = slack_clients["su"], slack_clients["cravatt"] + + # Before the invite, cravatt cannot read it. + assert cravatt.poll_channel_messages(cid, oldest="0") == [] + assert su.invite_to_channel(cid, [cravatt.bot_user_id]) is True + _post(su, cid, "after the invite") + assert "after the invite" in [ + m.get("text") for m in cravatt.poll_channel_messages(cid, oldest="0") + ], "the invited bot still cannot read the private channel" + + +def test_private_channels_are_excluded_from_the_public_listing( + slack_clients, private_channel, slack_list_all_channels +): + """Note the name: create_private_channel appends a UTC timestamp to whatever it is + given, because the reopen slug is deterministic per agent-pair + origin channel and + Slack rejects a duplicate with name_taken. The fixture returns the name Slack + actually assigned, not the one requested. + + Both halves go through the fully paginated listing. Asking `list_channels()` (one + 200-item page of 323) made the positive half a coin flip AND the negative half + vacuous — a private channel really leaking into the public listing would still be + absent from page 1 about 38% of the time, so `not in` proved nothing. + """ + name, cid = private_channel + su = slack_clients["su"] + assert slack_list_all_channels(su, include_private=True).get(name) == cid, ( + f"the private channel is missing from the include_private listing: {name}" + ) + assert name not in slack_list_all_channels(su, include_private=False), ( + "a private channel leaked into the public listing" + ) + + +def test_a_non_member_bot_posting_to_a_private_channel_is_reported( + slack_clients, private_channel +): + """With a visibility_lookup that knows the channel is private, the client must + raise BotNotInvitedToPrivateChannel rather than swallow the error — the point is + that an invite-path bug stays visible. + """ + name, cid = private_channel + from src.agent.slack_client import AgentSlackClient + + tok = os.environ["SLACK_TEST_BOT_TOKEN_WISEMAN"] + w = AgentSlackClient(agent_id="wiseman", bot_token=tok, + visibility_lookup=lambda c: "collab_private") + assert w.connect() is True + with pytest.raises(BotNotInvitedToPrivateChannel): + w.post_message(cid, "I was never invited") + + +def test_a_non_member_bot_without_the_lookup_degrades_quietly(slack_clients, private_channel): + """Control for the test above: the raise is conditional on the visibility lookup. + Without it the client cannot tell a private channel from a deleted one, and + returning None is the right degradation.""" + name, cid = private_channel + from src.agent.slack_client import AgentSlackClient + + tok = os.environ["SLACK_TEST_BOT_TOKEN_WISEMAN"] + w = AgentSlackClient(agent_id="wiseman", bot_token=tok) # no visibility_lookup + assert w.connect() is True + assert w.post_message(cid, "still not invited") is None diff --git a/tests/integration/test_slack_cohort_live.py b/tests/integration/test_slack_cohort_live.py new file mode 100644 index 0000000..d512e7d --- /dev/null +++ b/tests/integration/test_slack_cohort_live.py @@ -0,0 +1,353 @@ +"""The cohort gate and the Slack mirror together. + +`.notes/cohort-thorough-test-plan.md` excluded this block by instruction and noted it +was not testable anyway: no agent carried a bot token. With three probe bots it is. + +The claim that matters is a distinction the Slack-off suite structurally cannot make: +the gate filters **reads**, never **writes**. Every agent's message must reach Slack — +the channel is shared and a human reads it — while a gated agent must not act on it. +With NullTransport those two are the same observation. +""" + +import time +import uuid + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + Cohort, + CohortAuditEvent, + CohortMembership, + SimulationRun, +) +from src.visibility import VISIBILITY_COLLAB_PRIVATE, VISIBILITY_PUBLIC + +pytestmark = [pytest.mark.integration, pytest.mark.live_slack] + +AGENTS = ("su", "cravatt", "wiseman") +POST_GAP = 1.1 + + +@pytest.fixture +async def cohort_engine(engine, slack_clients, slack_probe_channel, monkeypatch): + import src.agent.simulation as sim + from src.config import get_settings as _real + + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + name, cid = slack_probe_channel + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [name]) + + patched = _real().model_copy(update={ + "cohort_isolation_enabled": True, "cohort_default_policy": "isolated", + "max_consecutive_reactive_turns": 3, "turn_delay_seconds": 0.0, + }) + monkeypatch.setattr(sim, "get_settings", lambda: patched) + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for aid in AGENTS: + db.add(AgentRegistry(agent_id=aid, bot_name=f"{aid.capitalize()}ProbeBot", + pi_name=f"PI {aid}", status="active")) + await db.commit() + + agents = [Agent(agent_id=a, bot_name=f"{a.capitalize()}ProbeBot", pi_name=f"PI {a}") + for a in AGENTS] + eng = SimulationEngine( + agents=agents, slack_clients=dict(slack_clients), budget_cap=0, + session_factory=factory, simulation_run_id=run_id, slack_enabled=True, + ) + eng.message_log.set_bot_name_map({f"{a}probebot": a for a in AGENTS}) + eng._bot_name_to_id = {f"{a}probebot": a for a in AGENTS} + eng.message_log.set_persist_callback(eng._enqueue_persist) + eng._channel_id_map = {name: cid} + eng._channel_visibility = {name: VISIBILITY_PUBLIC} + for a in eng.agents.values(): + a.state.subscribed_channels = {name} + a.state.last_seen_cursor = 0.0 + + yield eng, factory, run_id, name, cid + + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute(delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id)) + await db.execute(delete(AgentChannel).where(AgentChannel.simulation_run_id == run_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(AGENTS))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +async def _topology(factory, mapping): + async with factory() as db: + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + for cname, members in mapping.items(): + c = Cohort(name=cname) + db.add(c) + await db.flush() + for aid in members: + db.add(CohortMembership(cohort_id=c.id, agent_id=aid)) + await db.commit() + + +# --- T9.1 -------------------------------------------------------------------------- + + +async def test_the_gate_filters_reads_and_never_the_mirror(cohort_engine): + """The distinction Slack-off cannot make. + + su+cravatt share a cohort, wiseman is outside it. All three messages must reach + Slack — the channel is shared and a human reads it, so suppressing the WRITE would + be a bug, not the feature. Only su's *read* is filtered. + """ + eng, factory, run_id, name, cid = cohort_engine + await _topology(factory, {"alpha": ["su", "cravatt"], "beta": ["wiseman"]}) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su", "cravatt"} + assert eng.agents["wiseman"].allowed_sender_ids == {"wiseman"} + + for aid, text in (("su", "from su"), ("cravatt", "from cravatt"), + ("wiseman", "from wiseman")): + await eng._post_message(aid, name, text) + time.sleep(POST_GAP) + await eng._flush_persisted() + + # Every message is in Slack. The gate is a read filter, not a mute button. + live = [m.get("text") + for m in eng.slack_clients["su"].poll_channel_messages(cid, oldest="0")] + for text in ("from su", "from cravatt", "from wiseman"): + assert text in live, f"{text!r} never reached Slack: {live}" + + # And every row landed, un-gated (§6.2: ingestion is never gated). + async with factory() as db: + rows = (await db.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run_id))).scalars().all() + assert len(rows) == 3, [r.content for r in rows] + + # su's gated read excludes wiseman and includes its cohort-mate. + su = eng.agents["su"] + visible = {e.content for e in eng.message_log.get_new_top_level_posts( + since=0, channels={name}, exclude_agent_id="su", + allowed_sender_ids=su.allowed_sender_ids)} + assert visible == {"from cravatt"}, visible + + +# --- T9.2: mention stripping, observable only in Slack -------------------------------- + + +async def test_a_cross_cohort_mention_is_stripped_in_the_message_slack_receives( + cohort_engine +): + """The strip runs inside _post_message, so Slack is the only place its effect is + observable end to end. Both halves in ONE message: the outsider's mention is gone + and the cohort-mate's survives, so a strip that deleted every mention fails. + """ + eng, factory, run_id, name, cid = cohort_engine + await _topology(factory, {"alpha": ["su", "cravatt"]}) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su", "cravatt"} + + marker = uuid.uuid4().hex[:6] + await eng._post_message( + "su", name, + f"[{marker}] cc @CravattProbeBot and @WisemanProbeBot on this", + ) + time.sleep(POST_GAP) + await eng._flush_persisted() + + live = [m.get("text") + for m in eng.slack_clients["su"].poll_channel_messages(cid, oldest="0")] + posted = [t for t in live if marker in t] + assert posted, f"the message never reached Slack: {live}" + text = posted[0] + assert "WisemanProbeBot" not in text, ( + f"a cross-cohort mention survived into Slack: {text!r}" + ) + assert "@CravattProbeBot" in text, ( + f"the cohort-mate's mention was stripped too: {text!r}" + ) + assert eng._cohort_tags_stripped.get("su", 0) >= 1 + + # And the stored row matches what Slack shows — the strip is not display-only. + async with factory() as db: + row = (await db.execute(select(AgentMessage).where( + AgentMessage.simulation_run_id == run_id))).scalars().one() + assert "WisemanProbeBot" not in row.content + + +# --- T9.3: grandfathering across a restart, with Slack on ------------------------------- + + +async def test_a_cross_cohort_thread_is_grandfathered_and_still_replies_in_slack( + cohort_engine +): + """§8 calls the resumed run the normal path, because the DB rebuild reconstructs + threads gate-blind before the first recompute. This is the only test that exercises + that with Slack present. + """ + from src.agent.state import ThreadState + + eng, factory, run_id, name, cid = cohort_engine + await _topology(factory, {"alpha": ["su", "cravatt"]}) + await eng._recompute_allowed_sender_ids() + + await eng._post_message("su", name, "thread root") + time.sleep(POST_GAP) + await eng._flush_persisted() + async with factory() as db: + root = (await db.execute(select(AgentMessage).where( + AgentMessage.content == "thread root"))).scalars().one() + await eng._post_message("cravatt", name, "a reply", thread_ts=root.message_ts) + time.sleep(POST_GAP) + await eng._flush_persisted() + + su = eng.agents["su"] + su.state.active_threads[root.message_ts] = ThreadState( + thread_id=root.message_ts, channel=name, other_agent_id="cravatt", + message_count=2) + assert eng._owes_reply(su) is True, "precondition: the thread owes a reply in-cohort" + + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + await eng._recompute_allowed_sender_ids() + assert su.state.active_threads[root.message_ts].grandfathered is True + assert eng._owes_reply(su) is False, "a grandfathered thread must lose priority" + + # It may still conclude — and the reply must reach the real Slack thread. + await eng._post_message("su", name, "wrapping up", thread_ts=root.message_ts) + time.sleep(POST_GAP) + await eng._flush_persisted() + replies = eng.slack_clients["su"].get_thread_replies(cid, root.slack_ts) + assert "wrapping up" in [m.get("text") for m in replies], ( + "the grandfathered thread's concluding reply never reached Slack" + ) + + +# --- T9.4: private-channel polling needs a member bot ----------------------------------- + + +async def test_a_private_channel_is_polled_only_by_a_member_bot(cohort_engine, slack_clients): + """`_client_for_channel` picks a bot that is actually in the channel. A non-member + gets channel_not_found, so a wrong pick silently loses every message in the channel. + + Control: after inviting the second bot, its client does read it. + """ + eng, factory, run_id, name, cid = cohort_engine + su, wiseman = slack_clients["su"], slack_clients["wiseman"] + + priv = su.create_private_channel(f"t-priv-poll-{uuid.uuid4().hex[:6]}") + assert priv and priv.get("id"), priv + pname, pcid = priv["name"], priv["id"] + try: + eng._channel_id_map[pname] = pcid + eng._channel_visibility[pname] = VISIBILITY_COLLAB_PRIVATE + + su.post_message(pcid, "members only") + time.sleep(POST_GAP) + assert wiseman.poll_channel_messages(pcid, oldest="0") == [], ( + "a non-member read the private channel" + ) + + # Documented behaviour worth pinning: _client_for_channel keys ONLY on + # _private_channel_members, never on _channel_visibility. With the membership + # map empty it hands back the fallback even for a channel marked private, and + # that fallback then fails with channel_not_found on every poll tick. The real + # path (_sync_private_channels_from_db) populates both maps together, so this + # is a fail-soft rather than a defect — but the docstring's "returns None if + # the channel is private and no connected member is available" is only true + # once the map has been loaded. + assert eng._client_for_channel(pcid, wiseman) is wiseman + + # With membership known, the member bot is chosen. + eng._private_channel_members[pcid] = ["su"] + chosen = eng._client_for_channel(pcid, wiseman) + assert chosen is not None, "no member bot was found for a channel su is in" + assert "members only" in [ + m.get("text") for m in chosen.poll_channel_messages(pcid, oldest="0") + ], "the chosen client cannot read the channel it was chosen for" + + # And a private channel whose only member is disconnected yields None, so the + # caller skips it rather than erroring every tick. + eng._private_channel_members[pcid] = ["nobody"] + assert eng._client_for_channel(pcid, wiseman) is None + + # Control: invite wiseman and it can read it too. + assert su.invite_to_channel(pcid, [wiseman.bot_user_id]) is True + assert "members only" in [ + m.get("text") for m in wiseman.poll_channel_messages(pcid, oldest="0") + ] + finally: + su._call_with_retry(su._client.conversations_archive, channel=pcid) + + +async def test_the_private_channel_exemption_holds_over_slack(cohort_engine, slack_clients): + """§7 end to end with the mirror on: two agents in DIFFERENT cohorts, maximally + gated, still converse in the channel the PI made for them — and the messages are + really in Slack. + + Control: the same two agents' public traffic IS filtered from each other's reads, so + the private result cannot be explained by the gate being off. + """ + eng, factory, run_id, name, cid = cohort_engine + await _topology(factory, {"alpha": ["su"], "beta": ["cravatt"]}) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su"} + assert eng.agents["cravatt"].allowed_sender_ids == {"cravatt"} + + su, cravatt = slack_clients["su"], slack_clients["cravatt"] + priv = su.create_private_channel(f"t-priv-exempt-{uuid.uuid4().hex[:6]}") + assert priv and priv.get("id"), priv + pname, pcid = priv["name"], priv["id"] + try: + assert su.invite_to_channel(pcid, [cravatt.bot_user_id]) is True + eng._channel_id_map[pname] = pcid + eng._channel_visibility[pname] = VISIBILITY_COLLAB_PRIVATE + # cravatt posts to this channel by NAME below, and only su's client learned the + # id from create_private_channel. Without the shared cache, cravatt's + # _resolve_channel_id falls back to list_channels() — which never returns private + # channels at all in its default mode — and the raw name is handed to + # chat.postMessage. The engine shares the map for exactly this reason in + # production (_sync_private_channels_from_db / cache_channel_ids). + for c in slack_clients.values(): + c.cache_channel_ids({pname: pcid}) + for a in eng.agents.values(): + a.state.subscribed_channels.add(pname) + + await eng._post_message("cravatt", pname, "private: my angle") + time.sleep(POST_GAP) + await eng._post_message("su", name, "public: my angle") + time.sleep(POST_GAP) + await eng._flush_persisted() + + # Persisted with the right visibility, and really in Slack. + async with factory() as db: + byname = {r.channel_name: r for r in (await db.execute(select(AgentMessage) + .where(AgentMessage.simulation_run_id == run_id))).scalars().all()} + assert byname[pname].visibility == VISIBILITY_COLLAB_PRIVATE + assert byname[name].visibility == VISIBILITY_PUBLIC + assert "private: my angle" in [ + m.get("text") for m in su.poll_channel_messages(pcid, oldest="0")] + + # su is maximally gated yet sees cravatt's private-channel message. + sua = eng.agents["su"] + seen = {e.content for e in eng.message_log.get_new_top_level_posts( + since=0, channels={pname}, exclude_agent_id="su", + allowed_sender_ids=sua.allowed_sender_ids)} + assert seen == {"private: my angle"}, seen + + # Control: cravatt does NOT see su's public post. + cra = eng.agents["cravatt"] + pub = [e.content for e in eng.message_log.get_new_top_level_posts( + since=0, channels={name}, exclude_agent_id="cravatt", + allowed_sender_ids=cra.allowed_sender_ids)] + assert pub == [], f"control leg failed: the gate is not filtering public: {pub}" + finally: + su._call_with_retry(su._client.conversations_archive, channel=pcid) diff --git a/tests/integration/test_slack_lifecycle_live.py b/tests/integration/test_slack_lifecycle_live.py new file mode 100644 index 0000000..ff8478d --- /dev/null +++ b/tests/integration/test_slack_lifecycle_live.py @@ -0,0 +1,456 @@ +"""Restart, reconcile, Slack-off<->Slack-on transitions, and the error paths. + +T6, T10 and T11 of .notes/slack-integration-test-plan.md. + +The hybrid state is the one production is actually in: a run that started with Slack off +has DB-origin roots Slack has never seen, and then Slack comes on. Nothing tested that +combination, and it is where `_slack_parent_ts` earns its keep. +""" + +import time +import uuid + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.transport import NullTransport +from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + Cohort, + CohortAuditEvent, + CohortMembership, + SimulationRun, +) +from src.visibility import VISIBILITY_PUBLIC + +pytestmark = [pytest.mark.integration, pytest.mark.live_slack] + +AGENTS = ("su", "cravatt", "wiseman") +POST_GAP = 1.1 + + +@pytest.fixture +async def lifecycle(engine, slack_clients, slack_probe_channel, monkeypatch): + """A factory that can build engines repeatedly over ONE simulation_run_id, so a + restart is a genuinely new engine object against the same durable state.""" + import src.agent.simulation as sim + + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + name, cid = slack_probe_channel + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [name]) + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for aid in AGENTS: + db.add(AgentRegistry(agent_id=aid, bot_name=f"{aid.capitalize()}ProbeBot", + pi_name=f"PI {aid}", status="active")) + await db.commit() + + def build(*, slack_on: bool): + agents = [Agent(agent_id=a, bot_name=f"{a.capitalize()}ProbeBot", + pi_name=f"PI {a}") for a in AGENTS] + clients = (dict(slack_clients) if slack_on + else {a: NullTransport(a) for a in AGENTS}) + eng = SimulationEngine( + agents=agents, slack_clients=clients, budget_cap=0, + session_factory=factory, simulation_run_id=run_id, slack_enabled=slack_on, + ) + eng.message_log.set_bot_name_map({f"{a}probebot": a for a in AGENTS}) + eng._bot_name_to_id = {f"{a}probebot": a for a in AGENTS} + eng.message_log.set_persist_callback(eng._enqueue_persist) + eng._channel_id_map = {name: cid if slack_on else f"local:{name}"} + eng._channel_visibility = {name: VISIBILITY_PUBLIC} + for a in eng.agents.values(): + a.state.subscribed_channels = {name} + a.state.last_seen_cursor = 0.0 + return eng + + yield build, factory, run_id, name, cid, slack_clients + + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute(delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id)) + await db.execute(delete(AgentChannel).where(AgentChannel.simulation_run_id == run_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(AGENTS))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +async def _rows(factory, run_id): + async with factory() as db: + return (await db.execute( + select(AgentMessage).where(AgentMessage.simulation_run_id == run_id) + .order_by(AgentMessage.posted_at))).scalars().all() + + +# --- T6: restart and reconcile --------------------------------------------------------- + + +async def test_a_restart_rebuilds_the_log_without_duplicating_it(lifecycle): + """Two rebuild paths run at startup — the DB rebuild and the Slack reconcile — and + both append to the same log. `>0` would hide the failure that matters, so the count + is asserted exactly. + """ + build, factory, run_id, name, cid, _ = lifecycle + eng1 = build(slack_on=True) + await eng1._post_message("su", name, "before the restart") + time.sleep(POST_GAP) + await eng1._post_message("cravatt", name, "also before the restart") + time.sleep(POST_GAP) + await eng1._flush_persisted() + assert len(await _rows(factory, run_id)) == 2 + + eng2 = build(slack_on=True) + await eng2._rebuild_state_from_db() + await eng2._rebuild_state_from_slack() + + contents = sorted(e.content for e in eng2.message_log.get_new_top_level_posts( + since=0, channels={name}, exclude_agent_id="wiseman", allowed_sender_ids=None)) + assert contents == ["also before the restart", "before the restart"], contents + assert len(eng2.message_log) == 2, ( + f"the rebuild double-counted: {len(eng2.message_log)} entries for 2 messages" + ) + + +async def test_a_restart_restores_the_slack_mapping(lifecycle): + """Without slack_ts on the restored entries, `_slack_parent_ts` reports "no Slack + root" for every pre-restart thread and silently keeps all subsequent replies off + Slack. That is invisible from inside the DB.""" + build, factory, run_id, name, cid, _ = lifecycle + eng1 = build(slack_on=True) + await eng1._post_message("su", name, "root before restart") + time.sleep(POST_GAP) + await eng1._flush_persisted() + root = (await _rows(factory, run_id))[0] + assert root.slack_ts + + eng2 = build(slack_on=True) + await eng2._rebuild_state_from_db() + assert eng2._slack_parent_ts(root.message_ts) == root.slack_ts, ( + "the restored entry lost its Slack mapping — every later reply in this thread " + "would silently stop mirroring" + ) + + await eng2._post_message("cravatt", name, "reply after restart", + thread_ts=root.message_ts) + time.sleep(POST_GAP) + await eng2._flush_persisted() + live = eng2.slack_clients["su"].get_thread_replies(cid, root.slack_ts) + assert "reply after restart" in [m.get("text") for m in live] + + +async def test_a_restart_does_not_repost_to_slack(lifecycle): + """A reconcile that re-posted restored messages would double every message in the + channel — visible to the humans reading it, and invisible in our DB.""" + build, factory, run_id, name, cid, slack_clients = lifecycle + eng1 = build(slack_on=True) + await eng1._post_message("su", name, "posted once") + time.sleep(POST_GAP) + await eng1._flush_persisted() + before = len(slack_clients["su"].poll_channel_messages(cid, oldest="0")) + + eng2 = build(slack_on=True) + await eng2._rebuild_state_from_db() + await eng2._rebuild_state_from_slack() + time.sleep(POST_GAP) + + after = slack_clients["su"].poll_channel_messages(cid, oldest="0") + assert len(after) == before, ( + f"the restart posted {len(after) - before} extra message(s) to Slack" + ) + assert [m.get("text") for m in after].count("posted once") == 1 + + +async def test_ensure_seeded_channels_creates_a_missing_channel_with_a_live_client( + lifecycle, monkeypatch, slack_list_all_channels +): + """Only the Slack-off branch of _ensure_seeded_channels was covered. With a + connected client a missing channel must be created, get a real C… id, and really + exist in Slack (Rule S1 — the id in our map proves only that we stored an id).""" + import src.agent.simulation as sim + + build, factory, run_id, name, cid, slack_clients = lifecycle + fresh = f"t-seeded-{uuid.uuid4().hex[:8]}" + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [fresh]) + eng = build(slack_on=True) + try: + eng._ensure_seeded_channels() + first = eng._channel_id_map.get(fresh) + assert first and first.startswith("C"), ( + f"expected a real Slack channel id, got {first!r}" + ) + assert eng._channel_visibility[fresh] == VISIBILITY_PUBLIC + assert slack_list_all_channels(slack_clients["su"]).get(fresh) == first, ( + f"#{fresh} has an id in _channel_id_map but Slack has no such channel" + ) + finally: + if eng._channel_id_map.get(fresh, "").startswith("C"): + slack_clients["su"]._call_with_retry( + slack_clients["su"]._client.conversations_archive, + channel=eng._channel_id_map[fresh]) + + +async def test_ensure_seeded_channels_reuses_an_existing_channel( + lifecycle, monkeypatch, slack_list_all_channels +): + """The reuse branch: a second start must adopt the existing channel, not create a + second one. + + Discovery runs against the real `client.list_channels()`. It used to be patched to a + fully paginated ground truth because src read one 200-item page of a 323-channel + workspace, so this test passed or failed on whether Slack's id ordering happened to + put the channel we had just made inside that page — a ~62% coin flip, and the + original cause of this test's intermittent failures. Now that `list_channels` + paginates, the patch would only hide the code under test; the fully paginated fixture + is kept as the *independent* ground truth for "does Slack have this channel", which + is a different code path from the client's own listing on purpose. + """ + import src.agent.simulation as sim + + build, factory, run_id, name, cid, slack_clients = lifecycle + su = slack_clients["su"] + fresh = f"t-seeded-{uuid.uuid4().hex[:8]}" + made = su.create_channel(fresh) + assert made and made.get("id"), made + try: + ground = slack_list_all_channels(su) + assert ground.get(fresh) == made["id"] + for c in slack_clients.values(): + monkeypatch.setattr( + c, "create_channel", + lambda ch, _a=c.agent_id: pytest.fail( + f"[{_a}] _ensure_seeded_channels created #{ch} although Slack " + "already has it — a duplicate, not a reuse" + ), + ) + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [fresh]) + + eng = build(slack_on=True) + eng._ensure_seeded_channels() + assert eng._channel_id_map.get(fresh) == made["id"], ( + "the existing channel was not adopted: " + f"{eng._channel_id_map.get(fresh)!r} != {made['id']!r}" + ) + assert eng._channel_visibility[fresh] == VISIBILITY_PUBLIC + # And every client can address it without another listing round trip. + for c in slack_clients.values(): + assert c._channel_name_to_id.get(fresh) == made["id"], ( + f"[{c.agent_id}] did not get the shared channel map" + ) + finally: + su._call_with_retry(su._client.conversations_archive, channel=made["id"]) + + +async def test_ensure_seeded_channels_adopts_a_channel_beyond_the_first_page( + lifecycle, monkeypatch, slack_list_all_channels +): + """The pagination fix, observed through the engine's real bootstrap path. + + Was `xfail(strict=True)` while `list_channels` read a single 200-item page of + conversations.list and ignored `response_metadata.next_cursor`. A seeded channel + outside that page was treated as missing, conversations.create answered + `name_taken`, `create_channel` returned None, and the channel ended up with NO entry + in `_channel_id_map` — after which every post to it was addressed by name and Slack + answered `not_in_channel`. + + Deliberately not a coin flip. The victim is a channel that `list_channels()` reports + but conversations.list's FIRST PAGE does not — i.e. one the old implementation could + never see — and `create_channel` is replaced with a failure so the + duplicate-creation path is a hard error rather than something to infer from the + resulting id. + + The victim is cross-checked against an independent fully paginated walk before being + used, and the majority of the beyond-page-one set must check out. Neither walk is a + snapshot: conversations.list is cursor-paginated over a workspace this suite mutates, + and Slack's listing is eventually consistent, so any single channel can be missing + from one complete walk. Requiring a majority keeps the anti-invention claim (a client + fabricating names beyond page 1 fails) without letting Slack's index latency decide + the outcome. + """ + import src.agent.simulation as sim + + build, factory, run_id, name, cid, slack_clients = lifecycle + su = slack_clients["su"] + # One raw page: the whole of what src used to see. + first_page = { + ch["name"]: ch["id"] for ch in su._call_with_retry( + su._client.conversations_list, types="public_channel", limit=200, + )["channels"] + } + listed = su.list_channels() + ground = slack_list_all_channels(su) + + beyond = sorted(set(listed) - set(first_page)) + assert beyond, ( + f"list_channels() returned {len(listed)} channels and none of them is beyond " + "conversations.list's first 200-item page — it is not paginating, or the " + "workspace no longer exceeds one page" + ) + confirmed = [n for n in beyond if ground.get(n) == listed[n]] + assert len(confirmed) > len(beyond) // 2, ( + f"only {len(confirmed)} of {len(beyond)} channels beyond the first page could be " + "confirmed against an independent walk of Slack: " + f"{sorted(set(beyond) - set(confirmed))[:5]}" + ) + + victim = confirmed[0] + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [victim]) + for c in slack_clients.values(): + monkeypatch.setattr( + c, "create_channel", + lambda ch, _a=c.agent_id: pytest.fail( + f"[{_a}] _ensure_seeded_channels tried to create #{ch}, which Slack " + "already has beyond the first page of conversations.list — the " + "duplicate-creation path the pagination fix closed" + ), + ) + eng = build(slack_on=True) + eng._ensure_seeded_channels() + assert eng._channel_id_map.get(victim) == ground[victim], ( + f"#{victim} exists in Slack as {ground[victim]} but the engine mapped it to " + f"{eng._channel_id_map.get(victim)!r}" + ) + # And every client can address it by name, which is what the id is for. + for c in slack_clients.values(): + assert c._channel_name_to_id.get(victim) == ground[victim], ( + f"[{c.agent_id}] did not get the shared channel map" + ) + + +# --- T10: Slack-off <-> Slack-on --------------------------------------------------------- + + +async def test_off_then_on_keeps_old_rows_unmirrored_and_mirrors_new_ones(lifecycle): + """The realistic hybrid. Rows written while Slack was off must stay slack_ts NULL — + nothing may be retroactively invented — while new messages mirror normally.""" + build, factory, run_id, name, cid, slack_clients = lifecycle + + off = build(slack_on=False) + await off._post_message("su", name, "written while slack was off") + await off._flush_persisted() + rows = await _rows(factory, run_id) + assert len(rows) == 1 and rows[0].slack_ts is None, rows[0].slack_ts + canonical = rows[0].message_ts + + on = build(slack_on=True) + await on._rebuild_state_from_db() + await on._post_message("cravatt", name, "written after slack came on") + time.sleep(POST_GAP) + await on._flush_persisted() + + by_content = {r.content: r for r in await _rows(factory, run_id)} + assert by_content["written while slack was off"].slack_ts is None, ( + "a slack_ts was invented for a message Slack never saw" + ) + assert by_content["written after slack came on"].slack_ts, ( + "control leg failed: the mirror is not working at all after the transition" + ) + # The off-era root still has no Slack presence, so nothing threads against it. + assert on._slack_parent_ts(canonical) is None + live = [m.get("text") for m in slack_clients["su"].poll_channel_messages(cid, oldest="0")] + assert "written while slack was off" not in live + assert "written after slack came on" in live + + +async def test_on_then_off_stops_touching_slack_but_keeps_writing_rows(lifecycle): + """Control on the negative: assert the Slack channel does NOT grow, rather than + assuming "no Slack calls" from the absence of an error.""" + build, factory, run_id, name, cid, slack_clients = lifecycle + + on = build(slack_on=True) + await on._post_message("su", name, "while on") + time.sleep(POST_GAP) + await on._flush_persisted() + before = len(slack_clients["su"].poll_channel_messages(cid, oldest="0")) + + off = build(slack_on=False) + await off._rebuild_state_from_db() + await off._post_message("cravatt", name, "while off") + await off._flush_persisted() + time.sleep(POST_GAP) + + after = slack_clients["su"].poll_channel_messages(cid, oldest="0") + assert len(after) == before, ( + f"the Slack-off engine posted {len(after) - before} message(s) to Slack" + ) + contents = {r.content for r in await _rows(factory, run_id)} + assert contents == {"while on", "while off"}, ( + f"the DB is not the complete store while Slack is off: {contents}" + ) + + +# --- T11: failure modes ------------------------------------------------------------------ + + +async def test_a_revoked_token_degrades_to_slack_off_and_keeps_the_row(lifecycle): + """The one that matters most. The DB is the durable store, so a dead Slack token + must never cost a message — the turn completes and the row lands. + """ + build, factory, run_id, name, cid, _ = lifecycle + from src.agent.slack_client import AgentSlackClient + + eng = build(slack_on=True) + dead = AgentSlackClient(agent_id="su", bot_token="xoxb-0000-dead-token") + assert dead.connect() is False, "a bogus token must not authenticate" + assert dead.is_connected is False + eng.slack_clients["su"] = dead + + await eng._post_message("su", name, "posted with a dead token") + await eng._flush_persisted() + rows = [r for r in await _rows(factory, run_id) + if r.content == "posted with a dead token"] + assert rows, "the message was lost when Slack was unavailable" + assert rows[0].slack_ts is None, "a dead client must not report a Slack ts" + + +async def test_posting_to_an_archived_channel_does_not_crash(lifecycle, slack_clients): + """An archived channel is the state every one of this suite's own probe channels + ends in, so a stale id in _channel_id_map is not hypothetical.""" + build, factory, run_id, name, cid, _ = lifecycle + su = slack_clients["su"] + tmp = su.create_channel(f"t-archived-{uuid.uuid4().hex[:8]}") + assert tmp and tmp.get("id") + su._call_with_retry(su._client.conversations_archive, channel=tmp["id"]) + + eng = build(slack_on=True) + eng._channel_id_map[tmp["name"]] = tmp["id"] + eng._channel_visibility[tmp["name"]] = VISIBILITY_PUBLIC + await eng._post_message("su", tmp["name"], "into the archive") + await eng._flush_persisted() + + rows = [r for r in await _rows(factory, run_id) if r.content == "into the archive"] + assert rows, "the message was lost rather than kept in the DB" + assert rows[0].slack_ts is None + + +async def test_invite_tolerates_self_and_repeat_but_reports_a_real_failure(slack_clients): + """`cant_invite_self` and `already_in_channel` are successes by the documented + contract — "the invite is considered successful as long as every user ends up as a + member" — and the migration relies on that, since it invites both bots and one of + them created the channel. + + Control: a genuinely bad user id must still return False, or the tolerance would be + indistinguishable from a method that always returns True. + """ + su = slack_clients["su"] + ch = su.create_private_channel(f"t-selfinvite-{uuid.uuid4().hex[:6]}") + assert ch and ch.get("id") + try: + assert su.invite_to_channel(ch["id"], [su.bot_user_id]) is True + cravatt_id = slack_clients["cravatt"].bot_user_id + assert su.invite_to_channel(ch["id"], [cravatt_id]) is True + assert su.invite_to_channel(ch["id"], [cravatt_id]) is True, "already_in_channel" + assert su.invite_to_channel(ch["id"], []) is True, "an empty list is a no-op" + assert su.invite_to_channel(ch["id"], ["U000NOTREAL"]) is False, ( + "a genuine invite failure must be reported" + ) + finally: + su._call_with_retry(su._client.conversations_archive, channel=ch["id"]) diff --git a/tests/integration/test_slack_mirror_live.py b/tests/integration/test_slack_mirror_live.py new file mode 100644 index 0000000..bbb38c7 --- /dev/null +++ b/tests/integration/test_slack_mirror_live.py @@ -0,0 +1,359 @@ +"""The DB<->Slack mirror, against the real workspace. + +The highest-value file in the Slack plan: the last five commits on this branch were all +mirror fixes, and every one of them was invisible to the Slack-off suite (Rule S2). The +whole engine test suite runs with NullTransport, so a mirror that silently no-ops looks +identical from inside our own database. + +Rule S1 is enforced in every test here: an assertion on `agent_messages.slack_ts` is +paired with a read of that exact ts from Slack. + +| commit | fix | test | +|---|---|---| +| 10c240c | stop inferring slack_ts from the channel id | T5.4 | +| 7d8b177 | record the mirror mapping on polled bot messages | T5.3 | +| baa5583 | order "most recent" reads by posted_at | T5.5 | +| a93d136 | never hand a canonical id to Slack | T5.2 | +""" + +import time +import uuid + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.slack_client import ThreadNotFound +from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + Cohort, + CohortAuditEvent, + CohortMembership, + SimulationRun, +) +from src.visibility import VISIBILITY_PUBLIC + +pytestmark = [pytest.mark.integration, pytest.mark.live_slack] + +AGENTS = ("su", "cravatt", "wiseman") +POST_GAP = 1.1 + + +@pytest.fixture +async def slack_engine(engine, slack_clients, slack_probe_channel, monkeypatch): + """A real SimulationEngine with real Slack clients and slack_enabled=True. + + Everything the cohort suite does with NullTransport, but with the mirror live. The + probe channel is registered as the engine's only channel so nothing lands in a + seeded channel name. + """ + import src.agent.simulation as sim + + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + name, cid = slack_probe_channel + + # _poll_slack_for_pi_messages only polls channels whose name is in SEEDED_CHANNELS + # (or that are collab_private) — polling every public channel would sweep up + # archived channels from prior sims. The probe channel is neither, so without this + # the poller would silently skip it and every ingestion test would fail for a + # reason unrelated to the mirror. + monkeypatch.setattr(sim, "SEEDED_CHANNELS", [name]) + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + for aid in AGENTS: + db.add(AgentRegistry(agent_id=aid, bot_name=f"{aid.capitalize()}ProbeBot", + pi_name=f"PI {aid}", status="active")) + await db.commit() + + agents = [Agent(agent_id=a, bot_name=f"{a.capitalize()}ProbeBot", pi_name=f"PI {a}") + for a in AGENTS] + eng = SimulationEngine( + agents=agents, slack_clients=dict(slack_clients), budget_cap=0, + session_factory=factory, simulation_run_id=run_id, slack_enabled=True, + ) + eng.message_log.set_bot_name_map({f"{a}probebot": a for a in AGENTS}) + eng._bot_name_to_id = {f"{a}probebot": a for a in AGENTS} + eng.message_log.set_persist_callback(eng._enqueue_persist) + eng._channel_id_map = {name: cid} + eng._channel_visibility = {name: VISIBILITY_PUBLIC} + for a in eng.agents.values(): + a.state.subscribed_channels = {name} + a.state.last_seen_cursor = 0.0 + + yield eng, factory, run_id, name, cid + + async with factory() as db: + await db.execute(delete(CohortAuditEvent)) + await db.execute(delete(CohortMembership)) + await db.execute(delete(Cohort)) + await db.execute(delete(AgentMessage).where(AgentMessage.simulation_run_id == run_id)) + await db.execute(delete(AgentChannel).where(AgentChannel.simulation_run_id == run_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(AGENTS))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +async def _rows(factory, run_id): + async with factory() as db: + return (await db.execute( + select(AgentMessage).where(AgentMessage.simulation_run_id == run_id) + .order_by(AgentMessage.posted_at) + )).scalars().all() + + +async def _write_row(factory, run_id, **kw): + """A row written as if by another process — no Slack presence.""" + defaults = dict(simulation_run_id=run_id, channel_id="C_LOCAL", message_length=10, + phase="new_post", visibility=VISIBILITY_PUBLIC, is_bot=True, + thread_ts=None) + defaults.update(kw) + async with factory() as db: + db.add(AgentMessage(**defaults)) + await db.commit() + + +# --- T5.1 ----------------------------------------------------------------------------- + + +async def test_post_message_mirrors_and_records_a_usable_mapping(slack_engine): + """The row's slack_ts must name a message that really exists in Slack, in the + channel the row names. A mirror that wrote the row and skipped Slack, or posted to + Slack and skipped the row, fails one half.""" + eng, factory, run_id, name, cid = slack_engine + + await eng._post_message("su", name, "mirrored post") + time.sleep(POST_GAP) + await eng._flush_persisted() + + rows = await _rows(factory, run_id) + assert len(rows) == 1, [r.content for r in rows] + row = rows[0] + assert row.slack_ts, "no slack_ts recorded — the mirror silently no-oped" + assert row.slack_channel_id == cid, row.slack_channel_id + + live = {m["ts"]: m.get("text") + for m in eng.slack_clients["su"].poll_channel_messages(cid, oldest="0")} + assert row.slack_ts in live, ( + f"slack_ts {row.slack_ts} does not exist in Slack: {sorted(live)}" + ) + assert live[row.slack_ts] == "mirrored post" + + +async def test_a_threaded_reply_carries_the_parent_mapping(slack_engine): + eng, factory, run_id, name, cid = slack_engine + await eng._post_message("su", name, "thread root") + time.sleep(POST_GAP) + await eng._flush_persisted() + root = (await _rows(factory, run_id))[0] + + await eng._post_message("cravatt", name, "thread reply", thread_ts=root.message_ts) + time.sleep(POST_GAP) + await eng._flush_persisted() + + reply = [r for r in await _rows(factory, run_id) if r.content == "thread reply"][0] + assert reply.slack_thread_ts == root.slack_ts, ( + f"reply points at {reply.slack_thread_ts}, root is at {root.slack_ts}" + ) + live = eng.slack_clients["su"].get_thread_replies(cid, root.slack_ts) + assert "thread reply" in [m.get("text") for m in live], ( + "the reply is not in the Slack thread" + ) + + +# --- T5.2: a93d136 — never hand a canonical id to Slack --------------------------------- + + +async def test_a_db_origin_root_never_produces_a_phantom_slack_thread(slack_engine): + """A root minted while Slack was off has a canonical ts Slack has never seen. + Threading against it either errors or creates a phantom thread that no one can find. + + Control: a root that DOES have a slack_ts threads for real, so the skip is + conditional rather than the mirror having given up entirely. + """ + eng, factory, run_id, name, cid = slack_engine + + canonical = "9000.000100" + await _write_row(factory, run_id, agent_id="su", sender_name="SuProbeBot", + content="db-origin root", message_ts=canonical, posted_at=9000.0001, + channel_name=name, channel_id=cid) + await eng._poll_inbound_from_db() + assert eng._slack_parent_ts(canonical) is None, ( + "a canonical id with no slack_ts must not be offered to Slack" + ) + + await eng._post_message("cravatt", name, "reply to a db-origin root", + thread_ts=canonical) + time.sleep(POST_GAP) + await eng._flush_persisted() + # Slack raises thread_not_found for an id it never issued. That is stronger than an + # empty list: it proves no phantom thread exists rather than merely that it is + # empty. (If the mirror had posted, this would return the reply instead.) + with pytest.raises(ThreadNotFound): + eng.slack_clients["su"].get_thread_replies(cid, canonical) + # And the message is still durable in the DB — the mirror is skipped, not the write. + assert "reply to a db-origin root" in [ + r.content for r in await _rows(factory, run_id) + ], "the reply was lost entirely rather than merely not mirrored" + + # Control: a Slack-origin root threads for real. + await eng._post_message("su", name, "slack-origin root") + time.sleep(POST_GAP) + await eng._flush_persisted() + root = [r for r in await _rows(factory, run_id) if r.content == "slack-origin root"][0] + assert root.slack_ts + assert eng._slack_parent_ts(root.message_ts) == root.slack_ts + + await eng._post_message("cravatt", name, "real threaded reply", + thread_ts=root.message_ts) + time.sleep(POST_GAP) + await eng._flush_persisted() + live = eng.slack_clients["su"].get_thread_replies(cid, root.slack_ts) + assert "real threaded reply" in [m.get("text") for m in live], ( + "control leg failed: the mirror is not threading at all" + ) + + +# --- T5.3: 7d8b177 — polled bot messages get a mapping too -------------------------------- + + +async def test_a_polled_bot_message_records_its_mirror_mapping(slack_engine): + """A message posted by ANOTHER process's bot arrives via the Slack poller. Its row + must carry slack_ts/slack_channel_id, or a later reply to it cannot be threaded. + + Control: a human-authored message in the same poll must also land, so a poller that + dropped every bot message would not pass. + """ + eng, factory, run_id, name, cid = slack_engine + + # A bot message that this engine did NOT post: use cravatt's raw client directly, + # bypassing _post_message so nothing is written to the DB by us. + marker = f"posted out of band {uuid.uuid4().hex[:6]}" + out = eng.slack_clients["cravatt"].post_message(cid, marker) + time.sleep(POST_GAP) + assert out and out.get("ts") + + eng._last_channel_poll = 0.0 + await eng._poll_slack_for_pi_messages() + await eng._flush_persisted() + + rows = [r for r in await _rows(factory, run_id) if r.content == marker] + assert rows, ( + "the out-of-band bot message was never ingested. " + f"rows={[r.content for r in await _rows(factory, run_id)]}" + ) + row = rows[0] + assert row.slack_ts == out["ts"], ( + f"the polled bot message has slack_ts={row.slack_ts!r}, Slack says {out['ts']!r}" + ) + assert row.slack_channel_id == cid + assert row.is_bot is True, "a bot message was ingested as a human" + + +# --- T5.4: 10c240c — slack_ts is never inferred ------------------------------------------ + + +async def test_slack_ts_is_never_inferred_from_the_channel_id(slack_engine): + """A row with a slack_channel_id but no slack_ts means "we know the channel, we do + not know the message". Synthesising a ts from the channel id produces an identifier + Slack will reject or, worse, silently mis-thread against. + + Control: a row that DOES have a slack_ts returns it, so this is about the NULL case + and not about the reader being broken. + """ + from src.agent.simulation import _restored_slack_ts + + eng, factory, run_id, name, cid = slack_engine + + await _write_row(factory, run_id, agent_id="su", sender_name="SuProbeBot", + content="no slack presence", message_ts="9100.000100", + posted_at=9100.0001, channel_name=name, channel_id=cid, + slack_channel_id=cid, slack_ts=None) + rows = await _rows(factory, run_id) + assert _restored_slack_ts(rows[0]) is None, ( + f"a ts was invented for a row that has none: {_restored_slack_ts(rows[0])!r}" + ) + + await eng._post_message("su", name, "has slack presence") + time.sleep(POST_GAP) + await eng._flush_persisted() + real = [r for r in await _rows(factory, run_id) if r.content == "has slack presence"][0] + assert _restored_slack_ts(real) == real.slack_ts and real.slack_ts + + +# --- T5.5: baa5583 — "most recent" reads order by posted_at ------------------------------- + + +async def test_thread_history_is_ordered_by_posted_at_not_insertion(slack_engine): + """The DB poller and the Slack poller append independently, so insertion order can + disagree with real time. A scrambled thread is handed to the LLM verbatim. + + The rows are appended deliberately out of order; the assertion is the exact expected + sequence, not merely that the list is non-empty. + """ + eng, factory, run_id, name, cid = slack_engine + + await eng._post_message("su", name, "root") + time.sleep(POST_GAP) + await eng._flush_persisted() + root = (await _rows(factory, run_id))[0] + + # Two replies written straight to the DB, appended newest-first. + await _write_row(factory, run_id, agent_id="cravatt", sender_name="CravattProbeBot", + content="second reply", message_ts="9200.000200", posted_at=9200.0002, + channel_name=name, channel_id=cid, thread_ts=root.message_ts) + await eng._poll_inbound_from_db() + await _write_row(factory, run_id, agent_id="wiseman", sender_name="WisemanProbeBot", + content="first reply", message_ts="9200.000100", posted_at=9200.0001, + channel_name=name, channel_id=cid, thread_ts=root.message_ts) + await eng._poll_inbound_from_db() + + hist = [e.content for e in eng.message_log.get_thread_history(root.message_ts)] + assert hist == ["root", "first reply", "second reply"], ( + f"thread history is in insertion order, not posted_at order: {hist}" + ) + + +# --- T5.6: our own mirrored message must not come back as new ------------------------------ + + +async def test_polling_does_not_re_ingest_our_own_mirrored_message(slack_engine): + """The mapping exists so the poller can recognise our own post coming back. Without + it every mirrored message is re-ingested as a fresh inbound one, doubling the log + and giving other agents a phantom post to react to. + + Control: an out-of-band message posted between the two polls IS ingested, so this is + dedup and not a poller that stopped working. + """ + eng, factory, run_id, name, cid = slack_engine + + await eng._post_message("su", name, "mine, mirrored") + time.sleep(POST_GAP) + await eng._flush_persisted() + assert len(await _rows(factory, run_id)) == 1 + + for _ in range(2): + eng._last_channel_poll = 0.0 + await eng._poll_slack_for_pi_messages() + await eng._flush_persisted() + + rows = await _rows(factory, run_id) + assert [r.content for r in rows] == ["mine, mirrored"], ( + f"our own message was re-ingested: {[r.content for r in rows]}" + ) + assert len(eng.message_log) == 1, "the in-memory log double-counted" + + marker = f"someone else {uuid.uuid4().hex[:6]}" + eng.slack_clients["cravatt"].post_message(cid, marker) + time.sleep(POST_GAP) + eng._last_channel_poll = 0.0 + await eng._poll_slack_for_pi_messages() + await eng._flush_persisted() + assert marker in [r.content for r in await _rows(factory, run_id)], ( + "control leg failed: the poller ingests nothing at all" + ) diff --git a/tests/integration/test_slack_pi_live.py b/tests/integration/test_slack_pi_live.py new file mode 100644 index 0000000..68e9b34 --- /dev/null +++ b/tests/integration/test_slack_pi_live.py @@ -0,0 +1,190 @@ +"""PI interaction over real Slack, with the real classifier. + +T7 of the plan. These are `live_slack` AND `real_llm` — `handle_dm` routes on an LLM +classification, and the whole point is that the routing decision and the Slack delivery +are both real. + +What cannot be automated here: sending a message *as the human*. That needs the PI's own +user token, which we do not have. So the inbound half is driven by calling `handle_dm` +with the text directly — the same entry point the poller calls — and the outbound half +is asserted by reading the DM back out of Slack. +""" + +import os +import time +import uuid + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.agent.agent import Agent +from src.agent.message_log import MessageLog +from src.agent.pi_handler import PIHandler +from src.models import AgentRegistry, ResearcherProfile, SimulationRun, User + +pytestmark = [ + pytest.mark.integration, + pytest.mark.live_slack, + pytest.mark.real_llm, + pytest.mark.skipif(not os.environ.get("ANTHROPIC_API_KEY"), + reason="handle_dm classifies with a real LLM call"), +] + +POST_GAP = 1.1 + + +@pytest.fixture +async def pi_setup(engine, slack_clients, slack_pi_user_id): + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + u = User(id=uuid.uuid4(), orcid="9999-0000-0007-0001", + email="pi-live@scen.test", name="PI Su", + onboarding_complete=True, access_status="allowed") + # The PI<->Slack mapping is not a User column; the engine builds it in + # _load_pi_mappings and hands it to PIHandler explicitly, which is what the + # pi_slack_id_to_agent_ids argument below does. + db.add(u) + await db.flush() + db.add(AgentRegistry(agent_id="su", bot_name="SuProbeBot", pi_name="PI Su", + user_id=u.id, status="active")) + db.add(ResearcherProfile( + user_id=u.id, research_summary="CRISPR screens.", + techniques=["crispr"], keywords=["degrader"], + private_profile_md="# Private\nNo standing instructions yet.", + )) + await db.commit() + user_id = u.id + + agent = Agent(agent_id="su", bot_name="SuProbeBot", pi_name="PI Su") + agent._public_profile = "# Su Lab\n\nGenome-scale CRISPR screens.\n" + agent._private_profile = "No standing instructions yet." + log = MessageLog() + handler = PIHandler( + agents={"su": agent}, slack_clients={"su": slack_clients["su"]}, + pi_slack_id_to_agent_ids={slack_pi_user_id: ["su"]}, + message_log=log, session_factory=factory, simulation_run_id=run_id, + ) + yield handler, factory, run_id, user_id, slack_clients["su"], slack_pi_user_id + + async with factory() as db: + await db.execute(delete(ResearcherProfile).where( + ResearcherProfile.user_id == user_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id == "su")) + await db.execute(delete(User).where(User.id == user_id)) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +def _dm_texts(client, pi_user_id, since="0"): + dm = client.open_dm_channel(pi_user_id) + return [m.get("text", "") for m in client.poll_channel_messages(dm, oldest=since)] + + +async def test_a_question_dm_gets_a_real_reply_in_slack(pi_setup): + """The full round trip: a real classification, a real Opus answer, delivered to a + real Slack DM. Rule S1 — the reply is read back from Slack, not from a return value. + """ + handler, factory, run_id, user_id, client, pi = pi_setup + before = set(_dm_texts(client, pi)) + + await handler.handle_dm("su", pi, "What are you currently working on?") + time.sleep(POST_GAP) + + after = [t for t in _dm_texts(client, pi) if t not in before] + assert after, "the bot sent no DM at all in reply to a question" + assert len(" ".join(after)) > 40, f"the reply is suspiciously short: {after}" + + +async def test_a_standing_instruction_is_persisted_and_acknowledged(pi_setup): + """Both halves. The acknowledgement alone would be satisfied by a handler that + replied politely and wrote nothing; the DB write alone would be satisfied by one + that silently stored it and never told the PI. + """ + handler, factory, run_id, user_id, client, pi = pi_setup + async with factory() as db: + before = (await db.execute(select(ResearcherProfile.private_profile_md) + .where(ResearcherProfile.user_id == user_id))).scalar_one() + + marker = f"ferroptosis-{uuid.uuid4().hex[:6]}" + text = (f"From now on, always mention our interest in {marker} when proposing " + "collaborations.") + + # Classify first, and assert on it separately. handle_dm routes on a real LLM + # call, so a failure downstream has two possible causes — the classifier put this + # in the wrong bucket, or the persistence path is broken — and they need different + # fixes. Observed once in a full-suite run: the same instruction came back as + # `feedback` rather than `standing_instruction`, which is a classifier-boundary + # observation, not a persistence bug. Naming it here keeps the two apart. + cls = await handler._classify_dm(text) + routed = cls.get("category") + assert routed == "standing_instruction" or ( + routed == "feedback" and cls.get("implies_standing_instruction") + ), ( + f"the classifier routed an explicit 'from now on, always X' instruction to " + f"{routed!r} (implies_standing_instruction=" + f"{cls.get('implies_standing_instruction')!r}). That is a classifier-boundary " + "finding, not a persistence failure — the profile write below was never reached." + ) + + dm_before = set(_dm_texts(client, pi)) + await handler.handle_dm("su", pi, text) + time.sleep(POST_GAP) + + async with factory() as db: + after = (await db.execute(select(ResearcherProfile.private_profile_md) + .where(ResearcherProfile.user_id == user_id))).scalar_one() + assert after != before, "the standing instruction was not written to the profile" + assert marker in after, ( + f"the instruction was written but lost its content: {after[-400:]!r}" + ) + assert [t for t in _dm_texts(client, pi) if t not in dm_before], ( + "the PI was never told the instruction had been recorded" + ) + + +async def test_a_plain_question_does_not_become_a_standing_instruction(pi_setup): + """Control for the test above. A classifier that routed everything to + standing_instruction would pass it — and would quietly rewrite the PI's profile on + every question they ask. + """ + handler, factory, run_id, user_id, client, pi = pi_setup + async with factory() as db: + before = (await db.execute(select(ResearcherProfile.private_profile_md) + .where(ResearcherProfile.user_id == user_id))).scalar_one() + + await handler.handle_dm("su", pi, "Which channels are you currently in?") + time.sleep(POST_GAP) + + async with factory() as db: + after = (await db.execute(select(ResearcherProfile.private_profile_md) + .where(ResearcherProfile.user_id == user_id))).scalar_one() + assert after == before, ( + "a plain question rewrote the private profile — every question the PI asks " + "would silently become a standing instruction" + ) + + +async def test_notify_thread_conclusion_dms_the_pi(pi_setup): + """The outbound-only path. Asserted from Slack.""" + handler, factory, run_id, user_id, client, pi = pi_setup + before = set(_dm_texts(client, pi)) + marker = uuid.uuid4().hex[:6] + + from src.agent.state import ThreadState + + thread = ThreadState(thread_id="1.0", channel="t-probe", other_agent_id="cravatt", + message_count=4) + await handler.notify_thread_conclusion( + agent_id="su", thread=thread, outcome="proposal", + summary_text=f"Joint degrader screen [{marker}].", + ) + time.sleep(POST_GAP) + + new = [t for t in _dm_texts(client, pi) if t not in before] + assert new, "no conclusion DM was sent" + assert any(marker in t for t in new), ( + f"the conclusion DM does not carry the summary: {new}" + ) diff --git a/tests/integration/test_slack_private_live.py b/tests/integration/test_slack_private_live.py new file mode 100644 index 0000000..35f1a6d --- /dev/null +++ b/tests/integration/test_slack_private_live.py @@ -0,0 +1,208 @@ +"""Private-channel migration against the real workspace. + +T8 of the plan. `test_private_channel_migration.py` has 25 tests, all Slack-off. This +covers the half that only exists when Slack is on: the channel really gets created, both +bots really get invited, and the handover really lands — plus the parametrised +both-paths test for commit 2a2e98c. +""" + +import time +import uuid + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from src.models import ( + AgentChannel, + AgentMessage, + AgentRegistry, + SimulationRun, + ThreadDecision, + User, +) +from src.services.private_channels import migrate_public_thread_to_private +from src.visibility import VISIBILITY_COLLAB_PRIVATE, VISIBILITY_PUBLIC + +pytestmark = [pytest.mark.integration, pytest.mark.live_slack] + +POST_GAP = 1.1 +PAIR = ("su", "cravatt") + + +@pytest.fixture +async def migration_setup(engine, slack_clients, slack_bot_tokens): + """Two agents with real bot tokens on their registry rows, plus their PI users and + a concluded public thread ready to migrate.""" + factory = async_sessionmaker(engine, expire_on_commit=False) + run_id = uuid.uuid4() + created_channels = [] + # A unique origin channel per test. The private-channel slug is deterministic in + # (agent pair, origin channel) and create_private_channel only appends a + # second-granularity timestamp, so three tests sharing one origin name collide + # inside the same second and fall through to the name_taken retry — intermittently + # observed. Slack also treats ARCHIVED channel names as taken, so collisions + # accumulate across runs. A distinct origin per test is both stable and more + # faithful: each test is a different thread. + origin = f"t-origin-{uuid.uuid4().hex[:8]}" + + async with factory() as db: + db.add(SimulationRun(id=run_id, status="running")) + users = {} + for i, aid in enumerate(PAIR): + u = User(id=uuid.uuid4(), orcid=f"9999-0000-0008-{i:04d}", + email=f"{aid}-mig@scen.test", name=f"PI {aid.capitalize()}", + onboarding_complete=True, access_status="allowed") + db.add(u) + await db.flush() + users[aid] = u + db.add(AgentRegistry( + agent_id=aid, bot_name=f"{aid.capitalize()}ProbeBot", + pi_name=f"PI {aid.capitalize()}", user_id=u.id, status="active", + # The DB column is the authoritative token source — the migration + # service resolves both bots' clients from here. + slack_bot_token=slack_bot_tokens[aid], + )) + td = ThreadDecision( + simulation_run_id=run_id, thread_id="1700000000.000100", + channel=origin, agent_a=PAIR[0], agent_b=PAIR[1], + outcome="proposal", summary_text="A joint degrader screen.", + origin_visibility=VISIBILITY_PUBLIC, + ) + db.add(td) + await db.commit() + td_id, user_ids = td.id, {a: u.id for a, u in users.items()} + + yield factory, run_id, td_id, user_ids, created_channels, origin + + su = slack_clients["su"] + for cid in created_channels: + try: + su._call_with_retry(su._client.conversations_archive, channel=cid) + except Exception as exc: + print(f"WARNING: could not archive {cid}: {exc}") + async with factory() as db: + await db.execute(delete(AgentMessage)) + await db.execute(delete(AgentChannel)) + await db.execute(delete(ThreadDecision).where(ThreadDecision.simulation_run_id == run_id)) + await db.execute(delete(AgentRegistry).where(AgentRegistry.agent_id.in_(PAIR))) + await db.execute(delete(User).where(User.email.like("%-mig@scen.test"))) + await db.execute(delete(SimulationRun).where(SimulationRun.id == run_id)) + await db.commit() + + +async def test_migration_creates_a_real_private_channel_with_both_bots( + migration_setup, slack_clients, slack_list_all_channels +): + """The whole flow, asserted from Slack: the channel exists, is private, both bots + are members, and the handover text is really in it. + + Rule S1 — an AgentChannel row proves we wrote a row. + """ + factory, run_id, td_id, user_ids, created, origin = migration_setup + async with factory() as db: + td = (await db.execute(select(ThreadDecision).where( + ThreadDecision.id == td_id))).scalar_one() + creator = (await db.execute(select(User).where( + User.id == user_ids["su"]))).scalar_one() + result = await migrate_public_thread_to_private( + db, thread_decision=td, creator_agent_id="su", creator_pi_user=creator, + guidance_text="Focus on the ternary complex geometry first.", + ) + await db.commit() # the caller owns the transaction; see the test below + time.sleep(POST_GAP) + + cid = getattr(result, "channel_id", None) or result["channel_id"] + cname = getattr(result, "channel_name", None) or result["channel_name"] + created.append(cid) + + su, cravatt = slack_clients["su"], slack_clients["cravatt"] + assert su._is_private_channel(cid) is False or True # visibility_lookup unset here + members = su._call_with_retry(su._client.conversations_members, channel=cid)["members"] + assert su.bot_user_id in members, "the creating bot is not a member" + assert cravatt.bot_user_id in members, ( + f"the other bot was never invited: {members}" + ) + + texts = [m.get("text", "") for m in su.poll_channel_messages(cid, oldest="0")] + assert texts, f"the private channel #{cname} is empty — no handover was posted" + assert any("ternary complex" in t for t in texts), ( + f"the PI's guidance never reached the channel: {texts}" + ) + + # It is genuinely private: absent from the public listing, present in the private + # one. Both halves read the fully paginated listing, not AgentSlackClient's + # single-page one — with 323 public channels and a 200-item page, "in" was a coin + # flip and "not in" was vacuous. See tests/conftest.py::slack_list_all_channels. + assert cname not in slack_list_all_channels(su, include_private=False), ( + f"the private refinement channel #{cname} is in the public listing" + ) + assert slack_list_all_channels(su, include_private=True).get(cname) == cid, ( + f"#{cname} was reported by the migration but Slack does not list it" + ) + + +@pytest.mark.parametrize("slack_on", [True, False], ids=["slack-on", "slack-off"]) +async def test_the_handover_is_persisted_in_both_migration_paths( + migration_setup, slack_clients, monkeypatch, slack_on +): + """Commit 2a2e98c. The migration has two code paths — the Slack one and + `_migrate_offline` — and the handover message has to be written to agent_messages in + BOTH, or the simulation never ingests it and the refinement channel opens silent. + + Parametrised rather than two tests, so neither path can be quietly forgotten. + """ + factory, run_id, td_id, user_ids, created, origin = migration_setup + if not slack_on: + monkeypatch.setattr( + "src.services.private_channels._slack_enabled_for_migration", + lambda *a, **k: _false(), + ) + + async with factory() as db: + td = (await db.execute(select(ThreadDecision).where( + ThreadDecision.id == td_id))).scalar_one() + creator = (await db.execute(select(User).where( + User.id == user_ids["su"]))).scalar_one() + result = await migrate_public_thread_to_private( + db, thread_decision=td, creator_agent_id="su", creator_pi_user=creator, + guidance_text=f"Path marker {slack_on}.", + ) + # The service adds rows to the caller's session and leaves the commit to it — + # the reopen endpoint owns the transaction so the ProposalReview row it writes + # lands atomically with the migration. Without this the handover is rolled back. + await db.commit() + cid = getattr(result, "channel_id", None) or result["channel_id"] + if slack_on and cid and cid.startswith("C"): + created.append(cid) + + # NOT filtered by our run_id: _add_handover_message attaches to + # _latest_simulation_run_id(db), which is "the most recent run" rather than the + # thread's own — documented as intentional, because a web-UI migration happens + # between runs. Filtering on our run_id would make this assert for the wrong reason. + async with factory() as db: + rows = (await db.execute(select(AgentMessage))).scalars().all() + assert rows, f"[slack_on={slack_on}] no handover row was written to agent_messages" + + # Two groups, and both matter. The handover lands in the new private channel; a + # closing notice lands in the PUBLIC origin thread so the old conversation says + # where it went. Asserting "everything is collab_private" would have called that + # notice a bug. + private = [r for r in rows if r.channel_name != origin] + origin_rows = [r for r in rows if r.channel_name == origin] + assert private, f"[slack_on={slack_on}] nothing was written to the private channel" + assert all(r.visibility == VISIBILITY_COLLAB_PRIVATE for r in private), ( + f"[slack_on={slack_on}] a handover row is not collab_private: " + f"{[(r.channel_name, r.visibility) for r in private]}" + ) + assert origin_rows and all(r.visibility == VISIBILITY_PUBLIC for r in origin_rows), ( + f"[slack_on={slack_on}] the origin-thread notice is missing or mislabelled: " + f"{[(r.channel_name, r.visibility) for r in origin_rows]}" + ) + assert any(f"Path marker {slack_on}" in (r.content or "") for r in private), ( + f"[slack_on={slack_on}] the guidance text is missing from the handover" + ) + + +async def _false(): + return False diff --git a/tests/integration/test_slack_provision_live.py b/tests/integration/test_slack_provision_live.py new file mode 100644 index 0000000..de2f29a --- /dev/null +++ b/tests/integration/test_slack_provision_live.py @@ -0,0 +1,80 @@ +"""Live provisioning: the probe bots really exist in the workspace. + +This is the precondition every other live test rests on. If the tokens are dead, every +downstream failure would be about the tokens rather than about the code, so this runs +first and says so plainly. + +Rule S1: assert on Slack's answer, not on our database. A token column being set proves +we wrote a column. +""" + +import os + +import httpx +import pytest + +pytestmark = [pytest.mark.integration, pytest.mark.live_slack] + +SLACK_API = "https://slack.com/api" + + +def _auth_test(token: str) -> dict: + return httpx.post(f"{SLACK_API}/auth.test", + headers={"Authorization": f"Bearer {token}"}, timeout=15).json() + + +def test_every_probe_bot_authenticates_into_the_same_workspace(slack_bot_tokens_all): + # slack_bot_tokens_all, not slack_bot_tokens: this test compares the bots + # against each other, so a partial set makes it vacuous. Skip, don't fail. + slack_bot_tokens = slack_bot_tokens_all + assert set(slack_bot_tokens) == {"su", "cravatt", "wiseman"}, sorted(slack_bot_tokens) + teams, users = set(), {} + for aid, tok in slack_bot_tokens.items(): + d = _auth_test(tok) + assert d.get("ok"), f"{aid}: auth.test failed: {d.get('error')}" + teams.add(d["team_id"]) + users[aid] = d["user_id"] + assert len(teams) == 1, f"the bots are in different workspaces: {teams}" + assert teams == {os.environ["SLACK_TEST_TEAM_ID"]}, ( + f"installed into the wrong workspace: {teams}" + ) + assert len(set(users.values())) == 3, f"two agents share a bot user: {users}" + + +def test_lookup_team_id_agrees_with_auth_test(slack_bot_tokens): + """`lookup_team_id` is what start_provisioning uses to pin the OAuth URL to the + right workspace — the exact thing whose absence sent the first hand-built install + links at the wrong team.""" + from src.services.slack_provisioning import lookup_team_id + + tok = slack_bot_tokens["su"] + assert lookup_team_id(tok) == os.environ["SLACK_TEST_TEAM_ID"] + # Control: it must return None for a non-bot token rather than guessing. + assert lookup_team_id("xoxp-not-a-bot-token") is None + assert lookup_team_id("") is None + + +def test_the_granted_scopes_are_the_scopes_we_asked_for(slack_bot_tokens_all): + """apps.permissions.scopes reports what the install actually granted. + + su and cravatt were installed with groups:write; wiseman deliberately was not — it + carries exactly the BOT_SCOPES list as it shipped before this work. That asymmetry + is the live A/B behind test_private_channel_creation_needs_groups_write. + """ + def _scopes(tok): + # Every Slack response carries the token's granted scopes in this header. + # apps.permissions.scopes is the documented endpoint but returns + # `not_allowed_token_type` for granular-scope apps, which these are. + r = httpx.post(f"{SLACK_API}/auth.test", + headers={"Authorization": f"Bearer {tok}"}, timeout=15) + raw = r.headers.get("x-oauth-scopes") + assert raw, "Slack did not report the granted scopes" + return {s.strip() for s in raw.split(",") if s.strip()} + + su = _scopes(slack_bot_tokens_all["su"]) + wiseman = _scopes(slack_bot_tokens_all["wiseman"]) + assert "groups:write" in su, f"su was expected to have groups:write: {sorted(su)}" + assert "groups:write" not in wiseman, ( + "wiseman is the control for the missing-scope finding and must NOT have " + f"groups:write: {sorted(wiseman)}" + ) diff --git a/tests/integration/test_state_rebuild.py b/tests/integration/test_state_rebuild.py new file mode 100644 index 0000000..a921f73 --- /dev/null +++ b/tests/integration/test_state_rebuild.py @@ -0,0 +1,315 @@ +"""Restart fidelity of SimulationEngine._rebuild_agent_state, offline. + +`test_full_run_live.py::test_sigterm_and_restart_lose_nothing_and_duplicate_nothing` +covers this through a real SIGTERM against real Slack with real LLM turns. That +test needs workspace credentials and costs money, so the invariants it asserts +about *conversational state* are pinned here as well, with Slack off and no LLM: + +* an open thread stored in `agent_messages` before a restart is back in + `agent.state.active_threads` after the rebuild, exactly once (the live test's + "no open thread survived the restart" assertion); +* a thread with a `ThreadDecision` is NOT reopened (the live test's "a concluded + thread was reopened by the rebuild" assertion); +* running the rebuild twice changes nothing (`start()` calls it once today, so + this is the property that keeps a second caller from silently double-counting). + +The engine is driven at the same seam the live test's phase B uses — the real +`_rebuild_state_from_db()` then the real `_rebuild_agent_state()` — because +`_rebuild_agent_state` reads `self.message_log`, not `agent_messages`: the DB +pass is what puts the rows in the log, so testing the second without the first +would test a rebuild of an empty log. +""" + +import time +from datetime import UTC, datetime, timedelta + +import pytest + +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine +from src.agent.transport import NullTransport +from src.config import get_settings +from tests import factories + +pytestmark = pytest.mark.integration + +AGENT_IDS = ("su", "wiseman") + + +class _FrozenClock: + """Stand-in for the module-level `datetime` name in src.agent.simulation. + + Step 4b's cutoff is `datetime.now(UTC) - timedelta(...)`; by inspection, + neither `_rebuild_state_from_db` nor `_rebuild_agent_state` calls + `datetime` anywhere else, so stubbing just `.now()` pins the cutoff to an + exact instant and lets the boundary test assert `>=` inclusivity without + racing the real wall clock. + """ + + def __init__(self, fixed_now): + self._fixed_now = fixed_now + + def now(self, tz=None): + return self._fixed_now + + +class _FixtureSessionFactory: + """Route the engine's self-opened sessions at the rolled-back test session. + + Same shim as `test_message_persistence.py` uses, and for the same reason: the + rebuild does ``async with self.session_factory() as db:`` and must see the + rows this test wrote inside its own (rolled-back) transaction. __aexit__ must + NOT close the fixture-owned session. + """ + + def __init__(self, session): + self._s = session + + def __call__(self): + return self + + async def __aenter__(self): + return self._s + + async def __aexit__(self, *exc): + return False + + +def _engine_for(session, run_id, agent_ids=AGENT_IDS): + """A real SimulationEngine with Slack off and no budget.""" + agents = [ + Agent(agent_id=a, bot_name=f"{a.capitalize()}Bot", pi_name=f"PI {a}") + for a in agent_ids + ] + return SimulationEngine( + agents=agents, + slack_clients={a: NullTransport(a) for a in agent_ids}, + budget_cap=0, + session_factory=_FixtureSessionFactory(session), + simulation_run_id=run_id, + slack_enabled=False, + ) + + +async def _stored_thread(session, run, *, root="su", replier="wiseman", + channel="general", replies=3): + """Write one root post + `replies` replies as rows from a previous process. + + Timestamps are anchored to now: `_rebuild_state_from_db` windows the load to + REBUILD_WINDOW_S (14 days) OR-ed with "has no ThreadDecision", so an + epoch-1970 ts would be rescued by the OR clause in the open-thread tests and + silently dropped in the closed-thread one. Anchoring to now removes the + window as a variable. + """ + base = round(time.time(), 4) + root_ts = f"{base:.6f}" + await factories.make_agent_message( + session, run=run, agent_id=root, + channel_id="C1", channel_name=channel, + message_ts=root_ts, thread_ts=None, posted_at=base, + content=f"root post by {root}", sender_name=f"{root.capitalize()}Bot", + is_bot=True, + ) + for i in range(replies): + ts = f"{base + i + 1:.6f}" + await factories.make_agent_message( + session, run=run, agent_id=replier, + channel_id="C1", channel_name=channel, + message_ts=ts, thread_ts=root_ts, posted_at=base + i + 1, + content=f"reply {i} by {replier}", + sender_name=f"{replier.capitalize()}Bot", is_bot=True, + ) + await session.flush() + return root_ts + + +async def test_an_open_thread_survives_a_rebuild_exactly_once(db_session): + run = await factories.make_simulation_run(db_session) + root_ts = await _stored_thread(db_session, run, replies=3) + + eng = _engine_for(db_session, run.id) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + + su = eng.agents["su"] + assert list(su.state.active_threads) == [root_ts], ( + f"expected exactly the one open thread, got {list(su.state.active_threads)}" + ) + t = su.state.active_threads[root_ts] + assert t.other_agent_id == "wiseman" + assert t.channel == "general" + assert t.message_count == 4, ( + f"the whole thread must be restored, not just the root: {t.message_count}" + ) + assert t.has_pending_reply is True, ( + "the last message was the partner's, so su still owes a reply — losing this " + "is how a restart ghosts a conversation" + ) + # The partner side too: both participants track the thread. + assert list(eng.agents["wiseman"].state.active_threads) == [root_ts] + + # A second rebuild must be idempotent — restart is not always one-shot. + await eng._rebuild_agent_state() + assert list(su.state.active_threads) == [root_ts], ( + f"a second rebuild changed the thread set: {list(su.state.active_threads)}" + ) + assert su.state.active_threads[root_ts].message_count == 4 + + +async def test_a_decided_thread_is_not_reopened_by_a_rebuild(db_session): + """The live test asserts `not (restored & decided_a)`. Pinned offline.""" + run = await factories.make_simulation_run(db_session) + root_ts = await _stored_thread(db_session, run, replies=2) + await factories.make_thread_decision( + db_session, run=run, thread_id=root_ts, channel="general", + agent_a="su", agent_b="wiseman", outcome="no_proposal", + ) + await db_session.flush() + + eng = _engine_for(db_session, run.id) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + + assert eng.agents["su"].state.active_threads == {}, ( + "a thread with a ThreadDecision was reopened by the rebuild: " + f"{list(eng.agents['su'].state.active_threads)}" + ) + assert root_ts in eng._closed_thread_ids + + +async def test_a_second_rebuild_does_not_duplicate_restored_proposals(db_session): + """`pending_proposals` is a list and step 3 appends to it without clearing. + + Every unreviewed entry blocks the owning agent, so a duplicated one is not + cosmetic: it survives the single pop that reviewing it performs. + """ + run = await factories.make_simulation_run(db_session) + td = await factories.make_thread_decision( + db_session, run=run, thread_id="1500.000100", channel="general", + agent_a="su", agent_b="wiseman", outcome="proposal", + summary_text="a shared aim", + ) + await db_session.flush() + + eng = _engine_for(db_session, run.id) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + + su = eng.agents["su"] + assert [p.thread_id for p in su.state.pending_proposals] == [td.thread_id] + + await eng._rebuild_agent_state() + assert [p.thread_id for p in su.state.pending_proposals] == [td.thread_id], ( + "a second rebuild duplicated the restored proposal: " + f"{[p.thread_id for p in su.state.pending_proposals]}" + ) + + +async def test_a_second_rebuild_does_not_duplicate_prior_thread_context(db_session): + """`_prior_threads` is the Phase 5 dedup context, and step 1 appends to it. + + Duplicated entries are fed to the model as "you already discussed this N + times", which is a prompt corruption rather than a crash — so it needs a + test, not a reader. + """ + run = await factories.make_simulation_run(db_session) + await factories.make_thread_decision( + db_session, run=run, thread_id="1600.000100", channel="general", + agent_a="su", agent_b="wiseman", outcome="no_proposal", + summary_text="did not converge", + ) + await db_session.flush() + + eng = _engine_for(db_session, run.id) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + assert len(eng._prior_threads[("su", "wiseman")]) == 1 + + await eng._rebuild_agent_state() + assert len(eng._prior_threads[("su", "wiseman")]) == 1, ( + "a second rebuild duplicated the prior-thread dedup context: " + f"{eng._prior_threads[('su', 'wiseman')]}" + ) + + +async def test_call_times_rebuilds_from_the_window_and_api_call_count_stays_all_time( + db_session, monkeypatch, +): + """DB round trip through the REAL step 4b query — not a hand-built deque. + + The unit tests in test_hub_budget_scheduler.py::TestRestartRebuild hand-populate + `call_times` and re-check `_within_rate_limit`/`_turn_eligible`; they never invoke + the query itself. This test seeds `llm_call_logs` rows straddling the rate-limit + window boundary for one agent — including a row exactly ON the cutoff, to pin the + `>=` in the WHERE clause — and asserts the rebuilt `call_times` holds only the + in-window rows, oldest-first (`.order_by(created_at)` is load-bearing: the + rate limiter prunes with `popleft()` and assumes oldest-first). + + `api_call_count` must still reflect every row, including the out-of-window one: + step 4 (lifetime COUNT(*)) and step 4b (windowed call_times) read the same table + but must stay independent, or a restart would either bench an agent that isn't + actually over budget, or silently forgive one that is. + """ + run = await factories.make_simulation_run(db_session) + window = get_settings().llm_rate_window_seconds + frozen_now = datetime(2026, 1, 1, tzinfo=UTC) + cutoff = frozen_now - timedelta(seconds=window) + + outside = cutoff - timedelta(seconds=1) # just before cutoff: excluded + boundary = cutoff # exactly on cutoff: included (>=) + inside_older = cutoff + timedelta(seconds=100) + inside_newer = cutoff + timedelta(seconds=500) + + for ts in (outside, boundary, inside_older, inside_newer): + await factories.make_llm_call_log( + db_session, run=run, agent_id="su", created_at=ts, + ) + await db_session.flush() + + eng = _engine_for(db_session, run.id) + monkeypatch.setattr("src.agent.simulation.datetime", _FrozenClock(frozen_now)) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + + su = eng.agents["su"] + assert list(su.state.call_times) == pytest.approx([ + boundary.timestamp(), inside_older.timestamp(), inside_newer.timestamp(), + ]), ( + "call_times must hold only the in-window rows, oldest first: " + f"{list(su.state.call_times)}" + ) + # step 4's lifetime COUNT(*) counts all 4 rows, unaffected by the window + # filter that gated call_times above. + assert su.api_call_count == 4 + + +async def test_a_second_rebuild_does_not_duplicate_call_times(db_session, monkeypatch): + """Step 4b clears each agent's ledger before repopulating it. + + Same idempotency concern the pending_proposals and _prior_threads rebuilds above + document, applied to the sliding-window ledger: a plain, unguarded append would + duplicate every in-window entry on a second rebuild call and could throttle an + agent that is not actually over its allowance. Only one call site exists today + (`start()`), so this is latent, not live — pinned here so it stays that way. + """ + run = await factories.make_simulation_run(db_session) + frozen_now = datetime(2026, 1, 1, tzinfo=UTC) + await factories.make_llm_call_log( + db_session, run=run, agent_id="su", + created_at=frozen_now - timedelta(seconds=10), + ) + await db_session.flush() + + eng = _engine_for(db_session, run.id) + monkeypatch.setattr("src.agent.simulation.datetime", _FrozenClock(frozen_now)) + await eng._rebuild_state_from_db() + await eng._rebuild_agent_state() + + su = eng.agents["su"] + assert len(su.state.call_times) == 1 + + await eng._rebuild_agent_state() + assert len(su.state.call_times) == 1, ( + "a second rebuild duplicated the call_times ledger: " + f"{list(su.state.call_times)}" + ) diff --git a/tests/integration/test_worker.py b/tests/integration/test_worker.py new file mode 100644 index 0000000..5a05f58 --- /dev/null +++ b/tests/integration/test_worker.py @@ -0,0 +1,1072 @@ +"""T5 — the job-queue worker (`src/worker/main.py`) against a real Postgres. + +`src/worker/main.py` had zero coverage and it is what actually runs profile +generation in production: `claim_job`, `process_job`, `execute_generate_profile`, +`execute_monthly_refresh`, `run_worker`. + +Three things about the shape of this module: + +1. **The database is real and the sessions commit.** The shared `db_session` fixture + rolls back, which makes it useless here: `claim_job` commits, `process_job` commits, + and the whole question in T5.1 is what *another connection* sees. So these tests use + a committing `async_sessionmaker(engine)` (the pattern from + `tests/integration/test_cohort_engine_live.py`) and clean up after themselves. + `expire_on_commit=False` matches what `run_worker` builds. + +2. **Only the pipeline is mocked.** `src/worker/main.py` does + `from src.services.profile_pipeline import run_profile_pipeline` at import time, so + the effective binding is `src.worker.main.run_profile_pipeline` — patching + `src.services.profile_pipeline.run_profile_pipeline` would have no effect on the + worker. Every fake pipeline here writes a real `ResearcherProfile` through the real + session, because that is the "work" whose ordering against job completion T5.4 is + about. + +3. **What each test would catch.** Named per test in its docstring. Between them the + suite fails if `SKIP LOCKED` is deleted (T5.1b: `claim_job` blocks instead of + returning), if the retry cap is removed (T5.2: the claim/process loop does not + terminate; and the exhausted job gets claimed), and if completion is marked before + the work is done (T5.4: the in-flight probes see a completed job). +""" + +import asyncio +import time +import uuid +from contextlib import contextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace + +import pytest +from sqlalchemy import event, func, select, text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlalchemy.orm.attributes import set_committed_value + +from src.models import Job, ResearcherProfile, User +from src.worker import main as worker_main + +pytestmark = pytest.mark.integration + +# Everything this module writes is tagged so a crashed run can be swept next time. +TAG = "t5_worker" + + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + + +class _Harness: + """Committing session factory + row bookkeeping.""" + + def __init__(self, factory, pg_url, engine): + self.factory = factory + self.pg_url = pg_url + self.engine = engine + + async def sweep(self): + """Delete anything a previous (possibly interrupted) run of this file left. + + `users` cascades to jobs/profiles/publications; the extra jobs delete catches + jobs enqueued with `user_id = NULL`. + """ + async with self.factory() as db: + await db.execute(text("DELETE FROM jobs WHERE payload->>'tag' = :t"), {"t": TAG}) + await db.execute(text("DELETE FROM users WHERE orcid LIKE 'T5-%'")) + await db.commit() + + async def new_user(self, name="T5 PI") -> uuid.UUID: + uid = uuid.uuid4() + async with self.factory() as db: + db.add(User( + id=uid, + name=name, + orcid=f"T5-{uid.hex[:12]}", + email=f"t5-{uid.hex[:12]}@example.invalid", + access_status="allowed", + )) + await db.commit() + return uid + + async def enqueue( + self, + user_id: uuid.UUID | None = None, + job_type: str = "generate_profile", + max_attempts: int = 3, + enqueued_at: datetime | None = None, + ) -> uuid.UUID: + """Enqueue exactly the way `src/cli.py` and `src/routers/onboarding.py` do.""" + jid = uuid.uuid4() + payload = {"tag": TAG} + if user_id is not None: + payload["user_id"] = str(user_id) + async with self.factory() as db: + job = Job( + id=jid, + type=job_type, + user_id=user_id, + payload=payload, + max_attempts=max_attempts, + ) + if enqueued_at is not None: + job.enqueued_at = enqueued_at + db.add(job) + await db.commit() + return jid + + async def enqueue_for_missing_user( + self, ghost_id: uuid.UUID, job_type: str = "generate_profile" + ) -> uuid.UUID: + """A job whose payload names a user that does not exist. + + `jobs.user_id` is ON DELETE CASCADE, so deleting the user takes the job with it; + a payload pointing at a stranger is the reachable form of this state (and the + payload is what `execute_generate_profile` reads first). + """ + jid = await self.enqueue(None, job_type=job_type) + async with self.factory() as db: + job = (await db.execute(select(Job).where(Job.id == jid))).scalar_one() + job.payload = {"tag": TAG, "user_id": str(ghost_id)} + await db.commit() + return jid + + async def job(self, job_id: uuid.UUID) -> Job: + """A fresh read on its own connection — never the worker's session.""" + async with self.factory() as db: + return (await db.execute(select(Job).where(Job.id == job_id))).scalar_one() + + async def job_state(self, job_id: uuid.UUID): + async with self.factory() as db: + return (await db.execute( + select(Job.status, Job.attempts, Job.completed_at).where(Job.id == job_id) + )).one() + + async def profile_count(self, user_id: uuid.UUID) -> int: + async with self.factory() as db: + return await db.scalar( + select(func.count()) + .select_from(ResearcherProfile) + .where(ResearcherProfile.user_id == user_id) + ) + + async def foreign_pending_jobs(self) -> int: + async with self.factory() as db: + return await db.scalar(text( + "SELECT count(*) FROM jobs WHERE status = 'pending' " + "AND coalesce(payload->>'tag', '') <> :t" + ), {"t": TAG}) + + +@pytest.fixture +async def wk(engine, pg_url): + factory = async_sessionmaker(engine, expire_on_commit=False) + h = _Harness(factory, pg_url, engine) + await h.sweep() + yield h + await h.sweep() + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +async def _race_claims(factory, n: int): + """`n` concurrent `claim_job` calls, each on its own connection. + + The connection is warmed *before* the barrier so the race is over the claim + statement itself and not over connection setup. + + Returns `(claims, spans)`; `spans` is each call's (start, end) so the caller can + prove the calls actually overlapped. Without that, "exactly one worker claimed it" + is also what you get from six calls that happened to run one after another, and the + test would be asserting nothing about concurrency. + """ + barrier = asyncio.Barrier(n) + spans: list[tuple[float, float]] = [] + + async def one(): + async with factory() as db: + await db.execute(text("SELECT 1")) + await barrier.wait() + start = time.perf_counter() + job = await worker_main.claim_job(db) + spans.append((start, time.perf_counter())) + return job + + claims = await asyncio.gather(*[one() for _ in range(n)]) + return claims, spans + + +def _assert_overlapped(spans, n): + """All `n` calls were in flight at the same instant.""" + assert len(spans) == n + latest_start = max(s for s, _ in spans) + earliest_end = min(e for _, e in spans) + assert latest_start < earliest_end, ( + f"the {n} claim_job calls did not overlap in time (last start {latest_start}, " + f"first end {earliest_end}); they ran one after another, so this test says " + "nothing about concurrency" + ) + + +async def _one_round(factory) -> Job | None: + """One iteration of `run_worker`'s inner loop: claim, then process in a new session. + + Deliberately the same call shape as `run_worker`, including the arguments + `process_job` accepts and ignores. + """ + async with factory() as db: + job = await worker_main.claim_job(db) + if job is None: + return None + await worker_main.process_job(job.id, job.type, job.attempts, job.max_attempts, factory) + return job + + +async def _drain(factory, limit: int = 10) -> int: + """`_one_round` until the queue is empty. Returns the number of jobs processed. + + Bounded deliberately: if the retry cap is ever removed, a permanently failing job + is re-claimed forever, and that must surface as a loud failure rather than a hang. + """ + rounds = 0 + while rounds < limit: + if await _one_round(factory) is None: + return rounds + rounds += 1 + raise AssertionError( + f"the claim/process loop never terminated ({limit} rounds): a job is being " + "retried without limit, i.e. the attempts/max_attempts cap is gone" + ) + + +def _profile_writer(expected_users: dict[uuid.UUID, str]): + """A fake `run_profile_pipeline` that writes a real profile row for known users. + + Unknown users raise rather than silently succeeding, so a stray pending job from + somewhere else can never be mistaken for this test's work (and can never reach the + real ORCID/PubMed/Anthropic calls). + """ + + async def fake(user_id, db, job=None): + if user_id not in expected_users: + raise RuntimeError(f"T5: refusing to run the pipeline for unknown user {user_id}") + profile = ResearcherProfile( + user_id=user_id, research_summary=expected_users[user_id] + ) + db.add(profile) + await db.flush() + return profile + + return fake + + +async def _wait_until(pred, timeout=60.0, interval=0.1) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if await pred(): + return True + await asyncio.sleep(interval) + return False + + +# --------------------------------------------------------------------------- +# T5.1 — claim_job atomicity +# --------------------------------------------------------------------------- + + +async def test_claim_job_is_atomic_under_real_concurrency(wk): + """T5.1 — six concurrent workers, one pending job, exactly one claim. + + Positive control, in this test, per the plan: two jobs and two workers must yield + *both* jobs claimed. Without it "exactly one claim" is also satisfied by a + `claim_job` that never claims anything at all. + + Second control: the six calls are asserted to have genuinely overlapped in time. Six + serialized calls would also produce one claim and would prove nothing. + + `attempts == 1` is the third half: a lost update (two workers both reading + attempts=0 and both writing 1) would be invisible if only the claim count were + checked. + """ + uid = await wk.new_user() + jid = await wk.enqueue(uid) + + claims, spans = await _race_claims(wk.factory, n=6) + _assert_overlapped(spans, 6) + won = [c for c in claims if c is not None] + + assert len(won) == 1, ( + f"{len(won)} of 6 concurrent workers claimed the same job " + f"({[str(c.id) for c in won]}); the claim is not atomic" + ) + assert won[0].id == jid + state = await wk.job_state(jid) + assert state.status == "processing" + assert state.attempts == 1, ( + f"attempts={state.attempts} after a single claim — two workers incremented it " + "(lost update) even though only one returned the job" + ) + + # CONTROL: two jobs, two workers => both claimed. + # + # This is a parallel-progress control, not a SKIP LOCKED detector. Measured: with + # `skip_locked=False` substituted for the real thing, this assertion still passes, + # because Postgres' LockRows node re-checks the qual after the lock is granted and + # moves on to the next row when it no longer matches. Removing SKIP LOCKED is caught + # by the timing test below, and only by it. + uid2 = await wk.new_user() + j2 = await wk.enqueue(uid2) + j3 = await wk.enqueue(uid2) + + claims2, spans2 = await _race_claims(wk.factory, n=2) + _assert_overlapped(spans2, 2) + won2 = {c.id for c in claims2 if c is not None} + assert won2 == {j2, j3}, ( + f"two workers and two pending jobs claimed {len(won2)} job(s) ({won2}); " + "concurrent workers are not making progress in parallel" + ) + + +async def test_claim_job_skips_a_row_another_worker_holds_locked(wk): + """T5.1 — the `SKIP LOCKED` half, which the count-based test above cannot see. + + A worker that has claimed but not yet committed holds `FOR UPDATE` on its row. With + `SKIP LOCKED` a second worker steps over it *immediately*; with a plain `FOR UPDATE` + it blocks until the first commits. Both end at "exactly one claim", so the only + observable difference is time — hence the timeout, which is what makes this the test + that fails if someone deletes `skip_locked=True`. + + Control: the same call against the same row, once the lock is released, does claim + it. Otherwise `None` would prove nothing. + """ + uid = await wk.new_user() + jid = await wk.enqueue(uid) + + holder = wk.factory() + await holder.execute(text("SELECT id FROM jobs WHERE id = :i FOR UPDATE"), {"i": jid}) + try: + try: + async with wk.factory() as db: + blocked = await asyncio.wait_for(worker_main.claim_job(db), timeout=5.0) + except TimeoutError: + pytest.fail( + "claim_job blocked for 5s on a row another worker had locked: it waited " + "for the lock instead of skipping the row. SKIP LOCKED is gone, and a " + "worker pool now serialises behind whichever job is slowest." + ) + assert blocked is None, ( + f"claim_job returned job {blocked.id} while another transaction held " + "FOR UPDATE on it — two workers would run the same job" + ) + finally: + await holder.rollback() + await holder.close() + + # CONTROL: the row was claimable all along; the None above was the lock, not a + # claim_job that never claims. + async with wk.factory() as db: + got = await asyncio.wait_for(worker_main.claim_job(db), timeout=5.0) + assert got is not None and got.id == jid, ( + "claim_job did not claim an unlocked pending job, so the skip assertion above " + "was vacuous" + ) + + +# --------------------------------------------------------------------------- +# T5.2 — retries and the attempt cap +# --------------------------------------------------------------------------- + + +async def test_a_failing_job_retries_to_max_attempts_and_then_dies(wk, monkeypatch): + """T5.2 — attempts increments, and the job reaches a terminal state. + + Catches: removing the `job.attempts >= job.max_attempts` branch in `process_job` + (the job would end 'pending' forever); removing the cap altogether (`_drain` raises + rather than hanging). + + Control, in this test: a job that fails once and then succeeds ends 'completed' at + attempts=2 with its profile written — so "terminal state" is not being reached by a + retry mechanism that simply never retries. + """ + uid = await wk.new_user() + jid = await wk.enqueue(uid, max_attempts=3) + + seen_attempts = [] + + async def always_fails(user_id, db, job=None): + seen_attempts.append(job.attempts) + raise RuntimeError("pipeline exploded (T5.2)") + + monkeypatch.setattr(worker_main, "run_profile_pipeline", always_fails) + + rounds = await _drain(wk.factory, limit=10) + + assert rounds == 3, f"expected exactly max_attempts=3 executions, got {rounds}" + assert seen_attempts == [1, 2, 3], ( + f"attempts did not increment once per execution: {seen_attempts}" + ) + state = await wk.job_state(jid) + assert state.status == "dead", ( + f"a job that failed max_attempts times is {state.status!r}, not 'dead' — " + "nothing will ever move it out of the queue's way" + ) + assert state.attempts == 3 + row = await wk.job(jid) + assert "pipeline exploded (T5.2)" in (row.last_error or ""), ( + f"the failure reason was not recorded: {row.last_error!r}" + ) + assert await wk.profile_count(uid) == 0 + + # CONTROL: retrying is real work, not a state machine that gives up quietly. + uid2 = await wk.new_user() + jid2 = await wk.enqueue(uid2, max_attempts=3) + write = _profile_writer({uid2: "recovered on the second attempt"}) + calls = {"n": 0} + + async def fails_once(user_id, db, job=None): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("transient (T5.2 control)") + return await write(user_id, db, job) + + monkeypatch.setattr(worker_main, "run_profile_pipeline", fails_once) + rounds2 = await _drain(wk.factory, limit=10) + + assert rounds2 == 2, f"expected fail-then-succeed to take 2 executions, got {rounds2}" + state2 = await wk.job_state(jid2) + assert state2.status == "completed" and state2.attempts == 2 + assert await wk.profile_count(uid2) == 1, ( + "the retry was accounted for but the work never landed" + ) + + +async def test_claim_job_will_not_claim_a_job_whose_attempts_are_exhausted(wk): + """T5.2 — the cap's other enforcement point: the claim filter itself. + + `max_attempts=0` is the only way to reach `attempts >= max_attempts` while still + 'pending' (the failure path sets 'dead' at the same instant), and it is an input + value, not hand-written derived state — the column has no CHECK constraint and + defaults are applied at enqueue. + + Catches: deleting `Job.attempts < Job.max_attempts` from `claim_job`'s WHERE — the + exhausted job is enqueued *first*, so a claim that ignores the cap returns it. + + Control: a normal job enqueued a second later IS claimed, so "returns None" is not + the whole story. + """ + uid = await wk.new_user() + t0 = datetime.now(UTC) + exhausted = await wk.enqueue(uid, max_attempts=0, enqueued_at=t0) + normal = await wk.enqueue(uid, max_attempts=3, enqueued_at=t0 + timedelta(seconds=5)) + + async with wk.factory() as db: + first = await worker_main.claim_job(db) + assert first is not None, "nothing was claimed at all — the control job is missing" + assert first.id == normal, ( + "claim_job returned the job whose attempts are already exhausted " + "(max_attempts=0); the retry cap is not enforced at claim time and this job " + "will be picked up forever" + ) + + async with wk.factory() as db: + second = await worker_main.claim_job(db) + assert second is None, ( + f"claim_job claimed {second.id} — the exhausted job is still reachable" + ) + assert (await wk.job_state(exhausted)).status == "pending" + + +# --------------------------------------------------------------------------- +# T5.3 — a crashing job does not kill the worker +# --------------------------------------------------------------------------- + + +async def test_process_job_swallows_the_failure_so_the_next_job_still_runs(wk, monkeypatch): + """T5.3 (unit-of-the-loop half) — `process_job` must not propagate. + + If it raised, `run_worker`'s outer handler would catch it but the job's status would + never be written, leaving it stuck in 'processing' with no reaper. + + The crasher is given `max_attempts=1` so it reaches a terminal state in one round + and the queue moves on; the retry behaviour itself is T5.2's subject. + + Control: the second job, processed by the same loop right after, completes and + writes its profile. + """ + uid_bad = await wk.new_user("T5 crasher") + uid_good = await wk.new_user("T5 survivor") + t0 = datetime.now(UTC) + j_bad = await wk.enqueue(uid_bad, max_attempts=1, enqueued_at=t0) + j_good = await wk.enqueue(uid_good, enqueued_at=t0 + timedelta(seconds=5)) + + write = _profile_writer({uid_good: "survivor profile"}) + + async def crash_for_bad(user_id, db, job=None): + if user_id == uid_bad: + raise RuntimeError("kaboom (T5.3)") + return await write(user_id, db, job) + + monkeypatch.setattr(worker_main, "run_profile_pipeline", crash_for_bad) + + # Must return normally, not raise. + first = await _one_round(wk.factory) + assert first is not None and first.id == j_bad + assert (await wk.job_state(j_bad)).status == "dead" + + second = await _one_round(wk.factory) + assert second is not None and second.id == j_good, ( + f"the queue did not advance past the crashing job: claimed {second!r}" + ) + assert (await wk.job_state(j_good)).status == "completed" + assert await wk.profile_count(uid_good) == 1 + assert await wk.profile_count(uid_bad) == 0 + + +async def test_run_worker_loop_survives_a_crashing_job(wk, pg_url, monkeypatch): + """T5.3 — the real `run_worker` loop, not a reimplementation of it. + + This is the only test that executes `run_worker` itself: its engine construction, + its claim/process cycle, its idle sleep and its shutdown flag. The email + notification and inbound blocks are pushed out of reach with an absurd interval + rather than mocked, since email is out of scope for this plan. + + Control: the good job (enqueued *after* the crasher) reaching 'completed' is the + positive observation; "the loop did not raise" alone would pass for a loop that + exited immediately. + """ + dbname = pg_url.rsplit("/", 1)[-1].split("?")[0] + assert dbname != "copi", ( + f"refusing to run run_worker against {dbname!r}: this test drives the real " + "worker loop and it must never touch the live database" + ) + assert await wk.foreign_pending_jobs() == 0, ( + "there are pending jobs in this database that this file did not enqueue; " + "run_worker would claim them" + ) + + uid_bad = await wk.new_user("T5 loop crasher") + uid_good = await wk.new_user("T5 loop survivor") + t0 = datetime.now(UTC) + j_bad = await wk.enqueue(uid_bad, max_attempts=2, enqueued_at=t0) + j_good = await wk.enqueue(uid_good, enqueued_at=t0 + timedelta(seconds=5)) + + write = _profile_writer({uid_good: "loop survivor profile"}) + + async def crash_for_bad(user_id, db, job=None): + if user_id == uid_bad: + raise RuntimeError("kaboom in the loop (T5.3)") + return await write(user_id, db, job) + + monkeypatch.setattr(worker_main, "run_profile_pipeline", crash_for_bad) + monkeypatch.setattr(worker_main, "get_settings", lambda: SimpleNamespace( + database_url=pg_url, + worker_poll_interval=0.05, + notification_check_interval=10**9, + enable_inbound_email=False, + inbound_poll_interval=10**9, + )) + + worker_main._shutdown = False + task = asyncio.create_task(worker_main.run_worker()) + try: + async def good_done(): + return (await wk.job_state(j_good)).status == "completed" + + finished = await _wait_until(good_done, timeout=45.0) + finally: + worker_main._shutdown = True + try: + await asyncio.wait_for(task, timeout=30.0) + except TimeoutError: # pragma: no cover - only on a hung loop + task.cancel() + raise + + assert finished, ( + "run_worker never completed the job queued behind a crashing one: the loop " + "died, stalled, or is retrying the crasher forever" + ) + bad = await wk.job_state(j_bad) + assert bad.status == "dead" and bad.attempts == 2, ( + f"the crashing job ended {bad.status!r} after {bad.attempts} attempts" + ) + assert await wk.profile_count(uid_good) == 1 + assert await wk.profile_count(uid_bad) == 0 + + +# --------------------------------------------------------------------------- +# T5.4 — execute_generate_profile, and the ordering of completion vs the work +# --------------------------------------------------------------------------- + + +async def test_execute_generate_profile_calls_the_pipeline_with_the_claimed_job(wk, monkeypatch): + """T5.4 (wiring half) — the worker hands the pipeline the right user and job. + + `execute_generate_profile` resolves the user from `payload['user_id']`, verifies the + user exists, and passes the *job* through so the pipeline can record progress. The + fake writes progress exactly the way `run_profile_pipeline.update_progress` does; if + the job were not the session's live instance those writes would vanish, and + /onboarding — which renders `job.payload['progress']` — would show nothing forever. + + Control: the same worker run for a payload whose user does not exist must fail + loudly rather than complete, so "it called the pipeline" is not satisfied by a + worker that calls it for anybody. + """ + uid = await wk.new_user("T5 wiring") + jid = await wk.enqueue(uid) + seen = {} + + async def record(user_id, db, job=None): + seen["user_id"] = user_id + seen["job_id"] = job.id if job is not None else None + job.payload = dict(job.payload) + job.payload["progress"] = [{"step": "t5", "detail": "probe"}] + return await _profile_writer({uid: "wired"})(user_id, db, job) + + monkeypatch.setattr(worker_main, "run_profile_pipeline", record) + assert await _drain(wk.factory) == 1 + + assert seen["user_id"] == uid + assert seen["job_id"] == jid, "the pipeline was handed a different job than the claimed one" + assert (await wk.job(jid)).payload.get("progress") == [{"step": "t5", "detail": "probe"}], ( + "the job the pipeline was given is not the worker session's live instance, so " + "its progress writes were discarded" + ) + assert (await wk.job_state(jid)).status == "completed" + assert await wk.profile_count(uid) == 1 + + # CONTROL: a job pointing at a user that does not exist must not complete. + ghost = uuid.uuid4() + ghost_job = await wk.enqueue_for_missing_user(ghost) + + claimed = await _one_round(wk.factory) + assert claimed is not None and claimed.id == ghost_job + ghost_state = await wk.job_state(ghost_job) + assert ghost_state.status != "completed", ( + "a job for a nonexistent user was marked completed; the worker will happily " + "'generate a profile' for anyone" + ) + assert f"User {ghost} not found" in ((await wk.job(ghost_job)).last_error or "") + + +async def test_the_job_is_marked_completed_only_after_the_profile_row_exists(wk, monkeypatch): + """T5.4 (the ordering half) — the assertion that actually protects the data. + + A worker that writes `status='completed'` before the profile exists loses the + profile on a crash and never retries it. End-state assertions cannot see that: both + orderings finish with a completed job and a profile row. So the fake pipeline + observes the world *mid-execution*, from two vantage points: + + * the worker's own session (`job.status` on the tracked ORM object), and + * a second connection reading committed state. + + Catches: hoisting the `job.status = "completed"` / `db.commit()` block above the + dispatch, or committing the completion in a separate earlier transaction. + + The probes are proved non-vacuous by re-running the identical committed read after + `process_job` returns and seeing 'completed' there. + """ + uid = await wk.new_user("T5 ordering") + jid = await wk.enqueue(uid) + seen = {} + + async def observe_then_work(user_id, db, job=None): + seen["in_session_status"] = job.status + seen["committed_before"] = await wk.job_state(jid) + seen["profiles_before"] = await wk.profile_count(uid) + profile = ResearcherProfile(user_id=user_id, research_summary="ordering probe") + db.add(profile) + await db.flush() + # Flushed but not committed: still invisible to everyone else. + seen["profiles_after_flush"] = await wk.profile_count(uid) + return profile + + monkeypatch.setattr(worker_main, "run_profile_pipeline", observe_then_work) + assert await _drain(wk.factory) == 1 + + # The probe is capable of seeing a completed job — proved with the same query. + after = await wk.job_state(jid) + assert after.status == "completed" and after.completed_at is not None + assert await wk.profile_count(uid) == 1 + + assert seen["in_session_status"] == "processing", ( + f"the worker's own session had the job at {seen['in_session_status']!r} while " + "the pipeline was still running — completion is marked before the work" + ) + assert seen["committed_before"].status == "processing", ( + f"another connection saw the job as {seen['committed_before'].status!r} " + "mid-execution: the completion was committed before the profile existed, so a " + "crash here loses the profile permanently and the job is never retried" + ) + assert seen["committed_before"].completed_at is None + assert seen["profiles_before"] == 0 + assert seen["profiles_after_flush"] == 0, ( + "the profile became visible to other connections before the job was completed; " + "the two are no longer in one transaction" + ) + + +async def test_a_crash_after_partial_work_leaves_a_retryable_job(wk, monkeypatch): + """T5.4 (crash half) — the inverse of the ordering test. + + A pipeline that gets partway (profile row added and flushed) and then raises must + not leave the job 'completed'. It must be retryable. + + NOTE — observed side effect, reported not fixed: `process_job`'s except branch + commits the *same* session the pipeline was using, so the pipeline's partial writes + are committed alongside the failure record instead of being rolled back. This test + pins that behaviour rather than asserting the behaviour we would prefer; see the + task report. + + Control: the identical pipeline without the raise completes and is not retried. + """ + uid = await wk.new_user("T5 partial") + jid = await wk.enqueue(uid) + + async def half_then_crash(user_id, db, job=None): + db.add(ResearcherProfile(user_id=user_id, research_summary="half written")) + await db.flush() + raise RuntimeError("crashed after the profile row (T5.4)") + + monkeypatch.setattr(worker_main, "run_profile_pipeline", half_then_crash) + + async with wk.factory() as db: + job = await worker_main.claim_job(db) + await worker_main.process_job(job.id, job.type, job.attempts, job.max_attempts, wk.factory) + + state = await wk.job_state(jid) + assert state.status != "completed", ( + "a job whose pipeline raised is marked completed; the failure is now invisible" + ) + assert state.status == "pending", f"expected a retryable job, got {state.status!r}" + leaked = await wk.profile_count(uid) + + # CONTROL: the same shape of pipeline that does not raise does complete, so the + # assertions above are about the crash and not about the harness. + uid2 = await wk.new_user("T5 partial control") + jid2 = await wk.enqueue(uid2) + monkeypatch.setattr(worker_main, "run_profile_pipeline", _profile_writer({uid2: "whole"})) + assert await _drain(wk.factory, limit=10) >= 1 + assert (await wk.job_state(jid2)).status == "completed" + assert await wk.profile_count(uid2) == 1 + + # Characterization of the partial write described above. + assert leaked == 1, ( + "the partial profile write was rolled back — good, but the docstring and the " + "T5 report describing the except-branch commit are now out of date" + ) + + +async def test_a_database_error_in_the_pipeline_orphans_the_job_in_processing(wk, monkeypatch): + """T5.3/T5.4 — the one crash `process_job`'s error handling does not survive. + + Every other failure is caught, recorded and retried. A failure that poisons the + transaction is different: `process_job`'s except branch writes `last_error` and + commits *the same session*, and that commit raises in turn. The exception escapes + `process_job`, no status is written, and because `claim_job` only ever looks at + 'pending' rows the job is stranded in 'processing' with nothing in the system to + reap it. `run_worker`'s outer handler keeps the worker alive, so this is silent. + + The real pipeline reaches this shape at step 6 — `db.add(ResearcherProfile(...))` + then `flush()`, with `researcher_profiles.user_id` unique and no try/except — if two + generate_profile jobs for one user are ever in flight together. + + Characterization, reported not fixed. Control: the same claim/process pair with a + plain Python error does record 'pending' and is retried, so "stranded in processing" + is a property of the database error and not of the harness. + """ + uid = await wk.new_user("T5 db error") + async with wk.factory() as db: + db.add(ResearcherProfile(user_id=uid, research_summary="already here")) + await db.commit() + jid = await wk.enqueue(uid) + + async def duplicate_profile(user_id, db, job=None): + db.add(ResearcherProfile(user_id=user_id, research_summary="a second row")) + await db.flush() # unique violation on researcher_profiles.user_id + + monkeypatch.setattr(worker_main, "run_profile_pipeline", duplicate_profile) + + claimed = await _one_round_expecting_escape(wk.factory) + assert claimed is not None and claimed.id == jid + + state = await wk.job_state(jid) + assert state.status == "processing", ( + f"the job is {state.status!r}; if the error handler now survives a database " + "error this test should assert 'pending' and the bug report is stale" + ) + assert (await wk.job(jid)).last_error is None, ( + "the failure reason reached the row after all — the handler's commit succeeded" + ) + + # CONTROL: a non-database error through the identical path is recorded and retried. + uid2 = await wk.new_user("T5 db error control") + jid2 = await wk.enqueue(uid2) + + async def plain_error(user_id, db, job=None): + raise RuntimeError("a plain error (T5 control)") + + monkeypatch.setattr(worker_main, "run_profile_pipeline", plain_error) + claimed2 = await _one_round(wk.factory) + assert claimed2 is not None and claimed2.id == jid2 + state2 = await wk.job_state(jid2) + assert state2.status == "pending" + assert "a plain error (T5 control)" in ((await wk.job(jid2)).last_error or "") + + +async def _one_round_expecting_escape(factory) -> Job | None: + """`_one_round`, asserting that `process_job` raises rather than handling it.""" + async with factory() as db: + job = await worker_main.claim_job(db) + assert job is not None + with pytest.raises(Exception) as exc: # noqa: B017 - the type is the finding + await worker_main.process_job(job.id, job.type, job.attempts, job.max_attempts, factory) + assert "PendingRollbackError" in type(exc.value).__name__ or "rollback" in str(exc.value), ( + f"process_job raised {type(exc.value).__name__}: {exc.value}" + ) + return job + + +# --------------------------------------------------------------------------- +# T5.5 — execute_monthly_refresh +# --------------------------------------------------------------------------- + + +async def test_monthly_refresh_reruns_the_pipeline_without_duplicating_the_profile( + wk, monkeypatch +): + """T5.5 — what `execute_monthly_refresh` actually does today. + + It delegates to `execute_generate_profile`, i.e. re-runs the pipeline for the same + user. So the properties worth pinning are: it is dispatched at all (a + `monthly_refresh` job must not fall through to the unknown-type branch), it targets + the same user, and re-running does not create a second `ResearcherProfile`. + + It creates no follow-on job, and nothing anywhere in `src/` enqueues a + `monthly_refresh` — asserted here so that when scheduling is added, this test says + so rather than silently continuing to pass. + + Control for that absence assertion: the refresh job itself is present and was + processed, so "no new jobs" is not being satisfied by an empty table. + """ + uid = await wk.new_user("T5 refresh") + first = await wk.enqueue(uid, job_type="generate_profile") + monkeypatch.setattr(worker_main, "run_profile_pipeline", _profile_writer({uid: "v1"})) + assert await _drain(wk.factory) == 1 + assert (await wk.job_state(first)).status == "completed" + assert await wk.profile_count(uid) == 1 + + calls = [] + + async def refresh(user_id, db, job=None): + calls.append((user_id, job.type)) + existing = (await db.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == user_id) + )).scalar_one() + existing.research_summary = "v2 from the monthly refresh" + existing.profile_version = (existing.profile_version or 0) + 1 + await db.flush() + return existing + + monkeypatch.setattr(worker_main, "run_profile_pipeline", refresh) + refresh_job = await wk.enqueue(uid, job_type="monthly_refresh") + assert await _drain(wk.factory) == 1 + + assert calls == [(uid, "monthly_refresh")], ( + f"monthly_refresh did not reach the pipeline: {calls!r}" + ) + assert (await wk.job_state(refresh_job)).status == "completed" + assert await wk.profile_count(uid) == 1, ( + "the refresh created a second ResearcherProfile row instead of updating the " + "existing one" + ) + async with wk.factory() as db: + profile = (await db.execute( + select(ResearcherProfile).where(ResearcherProfile.user_id == uid) + )).scalar_one() + assert profile.research_summary == "v2 from the monthly refresh" + assert profile.profile_version == 1 + + async with wk.factory() as db: + jobs = (await db.execute(select(Job).where(Job.user_id == uid))).scalars().all() + assert {j.id for j in jobs} == {first, refresh_job}, ( + "the worker created follow-on job(s) — monthly_refresh now schedules work and " + "this test needs to describe it" + ) + + +async def test_monthly_refresh_for_a_missing_user_fails_loudly(wk, monkeypatch): + """T5.5 — the refresh path shares `execute_generate_profile`'s user check. + + Control in the same test: a refresh for a real user completes, so "did not complete" + is a property of the missing user and not of the refresh type being unsupported. + """ + called = [] + + async def should_not_run(user_id, db, job=None): + called.append(user_id) + raise AssertionError("the pipeline ran for a user that does not exist") + + monkeypatch.setattr(worker_main, "run_profile_pipeline", should_not_run) + + ghost = uuid.uuid4() + jid = await wk.enqueue_for_missing_user(ghost, job_type="monthly_refresh") + + claimed = await _one_round(wk.factory) + assert claimed is not None and claimed.id == jid + assert called == [] + state = await wk.job_state(jid) + assert state.status == "pending" + assert f"User {ghost} not found" in ((await wk.job(jid)).last_error or "") + + # CONTROL + uid = await wk.new_user("T5 refresh control") + ok = await wk.enqueue(uid, job_type="monthly_refresh") + monkeypatch.setattr(worker_main, "run_profile_pipeline", _profile_writer({uid: "ok"})) + assert await _drain(wk.factory, limit=10) >= 1 + assert (await wk.job_state(ok)).status == "completed" + + +async def test_a_job_with_no_user_at_all_fails_loudly(wk, monkeypatch): + """T5.5/T5.4 edge — `payload['user_id']` absent and `job.user_id` NULL. + + `execute_generate_profile` guards this with + `user_id_str = payload.get("user_id") or str(job.user_id)`, which for a NULL + user_id yields the string "None" — truthy — so the intended + `ValueError("Job missing user_id in payload")` is unreachable and the failure + arrives from `uuid.UUID("None")` instead. Reported, not fixed. The property that + matters is asserted first: the job does not complete. + """ + called = [] + + async def should_not_run(user_id, db, job=None): + called.append(user_id) + return None + + monkeypatch.setattr(worker_main, "run_profile_pipeline", should_not_run) + jid = await wk.enqueue(None) + + claimed = await _one_round(wk.factory) + assert claimed is not None and claimed.id == jid + assert called == [] + state = await wk.job_state(jid) + assert state.status != "completed" + last_error = (await wk.job(jid)).last_error or "" + assert last_error, "the job failed without recording why" + # Characterizes the unreachable guard described above. + assert "badly formed hexadecimal UUID string" in last_error, ( + f"the failure message changed to {last_error!r}; if the missing-user_id guard " + "is now reachable, this test should assert the intended message instead" + ) + + +# --------------------------------------------------------------------------- +# T5.6 — unknown job type +# --------------------------------------------------------------------------- + + +async def test_an_unknown_job_type_cannot_even_be_enqueued(wk): + """T5.6 (first line of defence) — `job_type_enum` rejects it at the database. + + This is also the reason the dispatcher test below has to doctor an in-memory + instance: there is no way to *store* an unknown type. + + Control: the identical raw insert with a legal type succeeds, so the rejection is + the enum and not a malformed statement. + """ + uid = await wk.new_user("T5 enum") + insert = text( + "INSERT INTO jobs (id, type, status, user_id, payload, attempts, max_attempts) " + "VALUES (:id, :type, 'pending', :uid, :payload, 0, 3)" + ) + payload = f'{{"tag": "{TAG}"}}' + + async with wk.factory() as db: + with pytest.raises(DBAPIError) as exc: + await db.execute(insert, { + "id": uuid.uuid4(), "type": "bogus_type", "uid": uid, "payload": payload, + }) + await db.rollback() + assert "job_type_enum" in str(exc.value), ( + f"the insert failed for some reason other than the enum: {exc.value}" + ) + + # CONTROL + legal = uuid.uuid4() + async with wk.factory() as db: + await db.execute(insert, { + "id": legal, "type": "generate_profile", "uid": uid, "payload": payload, + }) + await db.commit() + assert (await wk.job_state(legal)).status == "pending" + + +@contextmanager +def _job_type_forced_on_load(job_id, job_type): + """Make the worker's own `select(Job)` load `job_id` with `type = job_type`. + + An unknown type cannot be stored (the test above shows `job_type_enum` rejecting + it), so the only way to reach `process_job`'s else-branch is to change what the + dispatcher reads. The mapper-level `load` event fires as `process_job` re-fetches + the job in its real session, and `set_committed_value` writes the value as if it had + come from the row — so the attribute is not dirty and the subsequent UPDATE never + tries to push the illegal value back through the enum. + + Nothing about the worker is stubbed: it is the real session, the real query, the + real dispatch. + """ + + def _on_load(target, _context): + if target.id == job_id: + set_committed_value(target, "type", job_type) + + event.listen(Job, "load", _on_load) + try: + yield + finally: + event.remove(Job, "load", _on_load) + + +async def test_an_unknown_job_type_is_rejected_loudly_by_the_dispatcher(wk, monkeypatch): + """T5.6 — `process_job`'s else-branch must not silently succeed. + + Control: the same harness with a legal type runs the pipeline and completes — so a + failure above is the unknown type and not the doctoring mechanism. + """ + uid = await wk.new_user("T5 unknown type") + jid = await wk.enqueue(uid) + ran = [] + + async def pipeline(user_id, db, job=None): + ran.append(user_id) + return await _profile_writer({uid: "should not happen"})(user_id, db, job) + + monkeypatch.setattr(worker_main, "run_profile_pipeline", pipeline) + + with _job_type_forced_on_load(jid, "bogus_type"): + await worker_main.process_job(jid, "bogus_type", 0, 3, wk.factory) + + assert ran == [], "an unknown job type reached the profile pipeline" + state = await wk.job_state(jid) + assert state.status != "completed", ( + "a job of an unknown type was marked completed — the worker silently did " + "nothing and reported success" + ) + row = await wk.job(jid) + assert "Unknown job type: bogus_type" in (row.last_error or ""), ( + f"the rejection was not recorded: {row.last_error!r}" + ) + assert row.type == "generate_profile", ( + "the doctored type was written back to the database, which the enum should " + "have made impossible" + ) + assert await wk.profile_count(uid) == 0 + + # CONTROL: the same doctoring mechanism with a legal type completes the job. + uid2 = await wk.new_user("T5 unknown type control") + jid2 = await wk.enqueue(uid2) + monkeypatch.setattr(worker_main, "run_profile_pipeline", _profile_writer({uid2: "ok"})) + with _job_type_forced_on_load(jid2, "generate_profile"): + await worker_main.process_job(jid2, "generate_profile", 0, 3, wk.factory) + assert (await wk.job_state(jid2)).status == "completed" + assert await wk.profile_count(uid2) == 1 diff --git a/tests/live_api/__init__.py b/tests/live_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/live_api/test_grants_live.py b/tests/live_api/test_grants_live.py new file mode 100644 index 0000000..6a23679 --- /dev/null +++ b/tests/live_api/test_grants_live.py @@ -0,0 +1,616 @@ +"""grants.gov Search2 / fetchOpportunity, live. + +Rule L2 governs this file more than any other in the tier: **every funding opportunity +eventually closes.** Nothing here may assert that a particular `opp_id`, FOA number or +title is present or open. Every id used by an assertion is taken from the same live +response the assertion checks, so the tests are self-updating and cannot go stale. + +Rule L1 is why the two `..._contract_..._fixture_still_matches_...` tests exist. +`tests/contract/test_grants_contract.py` builds its opportunities from HAND-WRITTEN +dicts — literals, not recorded responses. All 10 of those tests would still pass if +grants.gov renamed or dropped a field, and production would break silently. Rather than +copy those literals here (which would only duplicate the same belief), the drift tests +parse the contract module's source and walk the dicts it actually asserts on, so editing +the fixture moves the drift check with it. + +Rule L3: grants.gov answers HTTP 200 with `errorcode: 0` and `msg: "Webservice +Succeeds"` even when its own backend is unavailable — the tell is a `data` block of +`{serverURI, message}` where an opportunity should be. Every assertion below is worded +to separate provider-down / rate-limited / schema-changed / our-parser-broken. +""" + +import ast +import pathlib + +import httpx +import pytest + +from src.agent.grantbot import _parse_close_date +from src.services import grants + +pytestmark = [pytest.mark.live_api] + +_CONTRACT_FILE = ( + pathlib.Path(__file__).resolve().parents[1] / "contract" / "test_grants_contract.py" +) + +# The keys `search_opportunities`/`list_posted_opportunities` promise their callers. +LIST_KEYS = {"id", "number", "title", "agency", "open_date", "close_date"} +SEARCH_KEYS = LIST_KEYS | {"description"} + +# camelCase keys mark a contract literal as API-shaped. The same module also contains +# snake_case dicts — the *expected output* of our mapper — and comparing those against +# grants.gov would report drift on field names grants.gov never had. +_API_SHAPED = frozenset({ + "agencyCode", "openDate", "closeDate", "awardCeiling", "awardFloor", + "categoryOfFundingActivity", "eligibleApplicants", "additionalInformationUrl", + "synopsis", +}) + +# Not an FOA prefix any agency issues, and confirmed to return zero hits. +BOGUS_NUMBER = "ZZZ-QQ-99-999" +# Pure nonsense: no English stem, so a working search must return nothing. ("nonsense" +# phrases built from real words like "termite" do match, and would make the control +# pass for the wrong reason.) +GIBBERISH = ["qxzjvbnp plurmfk", "zzqqxxwvv"] + +_PAGE_SIZE = 250 # must match list_posted_opportunities' internal page size + + +def key_paths(obj, prefix="", empty=None): + """Every dotted key path in a nested dict/list structure. + + ``empty`` collects the paths of containers that are present but EMPTY. That + distinction is the whole difference between "grants.gov renamed a field" and "this + particular opportunity has nothing in that list", and conflating them makes the + drift check cry wolf. + """ + if isinstance(obj, dict): + if not obj: + if empty is not None: + empty.add(prefix) + yield prefix + return + for k, v in obj.items(): + yield from key_paths(v, f"{prefix}.{k}" if prefix else k, empty) + elif isinstance(obj, list): + if obj: + yield from key_paths(obj[0], f"{prefix}[]", empty) + else: + if empty is not None: + empty.add(f"{prefix}[]") + yield f"{prefix}[]" + else: + yield prefix + + +def contract_fixture_literals() -> dict[str, list[dict]]: + """The API-shaped dict literals `tests/contract/test_grants_contract.py` asserts on. + + Read out of that file's source rather than re-declared here: a second hand-written + copy would drift from the first and the drift test would then be checking my belief + about their belief. Split into "search" and "detail" by the enclosing test's name, + because the two endpoints return different shapes and only one of them can be + verified when the detail backend is down. + """ + tree = ast.parse(_CONTRACT_FILE.read_text()) + out: dict[str, list[dict]] = {"search": [], "detail": []} + for fn in ast.walk(tree): + if not isinstance(fn, ast.FunctionDef | ast.AsyncFunctionDef): + continue + for node in ast.walk(fn): + if not isinstance(node, ast.Dict): + continue + try: + value = ast.literal_eval(node) + except (ValueError, SyntaxError): + continue # not a pure literal (e.g. the envelope built by _search_payload) + if not isinstance(value, dict) or "number" not in value: + continue + if not _API_SHAPED & set(value): + continue # a mapped-output expectation, not an API shape + out["detail" if "detail" in fn.name else "search"].append(value) + return out + + +async def _raw_post(url: str, payload: dict) -> dict: + """A direct call, bypassing our parser — the drift tests must see grants.gov's own + JSON, and the classification helpers must see the envelope our parser discards.""" + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post(url, json=payload) + except httpx.HTTPError as exc: # pragma: no cover - network + pytest.fail(f"grants.gov {url} was unreachable ({exc!r}) — provider down or the " + "container has no egress; this is not a schema change") + assert resp.status_code == 200, ( + f"grants.gov {url} answered HTTP {resp.status_code} — provider down or rate " + f"limited (we treat grants.gov as 1 req/s), not a schema change. " + f"Body: {resp.text[:300]!r}" + ) + return resp.json() + + +def detail_backend_outage(raw: dict) -> str | None: + """grants.gov's own "my backend is down" body, or None if this is a real response. + + fetchOpportunity returns HTTP 200 / errorcode 0 / msg "Webservice Succeeds" and puts + `{serverURI, message}` in `data` when the apply07 backend is unavailable. Without + this check, `fetch_opportunity_detail` returning None during an outage is + indistinguishable from our parser dropping a valid opportunity (Rule L3). + """ + data = raw.get("data") + if isinstance(data, dict) and "serverURI" in data and "number" not in data: + return str(data.get("message") or data)[:300] + return None + + +# --------------------------------------------------------------------------- T3.1 + + +async def test_list_posted_opportunities_returns_a_wellformed_page(api_budget): + """Shape, key set, paging arithmetic and date parseability — never an opportunity. + + Control: the raw probe supplies `hitCount` independently, so "we got a lot of rows" + cannot be satisfied by a pager that silently stopped after page one, and cannot be + called a failure when grants.gov genuinely has few postings. + """ + agencies = grants.BIOMEDICAL_AGENCIES + api_budget.wait("grants") + probe = await _raw_post(grants.SEARCH_URL, { + "oppStatuses": "posted", + "agencies": "|".join(agencies), + "rows": 1, + "startRecordNum": 0, + }) + hit_count = probe.get("data", {}).get("hitCount") + assert isinstance(hit_count, int) and hit_count > 0, ( + "grants.gov search2 did not return an integer data.hitCount for posted " + f"{agencies} opportunities — either the envelope changed shape (schema) or the " + f"provider is degraded. Got: {probe.get('data', {}).get('hitCount')!r}" + ) + + # Charge the budget for every page list_posted_opportunities is about to request; + # it paginates internally and never sees the rate limiter. + for _ in range(min(hit_count // _PAGE_SIZE + 1, 10)): + api_budget.wait("grants") + listed = await grants.list_posted_opportunities() + + assert listed, ( + f"grants.gov reports {hit_count} posted {agencies} opportunities but " + "list_posted_opportunities parsed none — the data.oppHits path is broken " + "(schema change or parser), not an empty catalogue" + ) + for item in listed[:50]: + assert set(item) == LIST_KEYS, ( + "list_posted_opportunities' mapped keys changed — callers " + f"(agent/grantbot.py, agent/tools.py) read {sorted(LIST_KEYS)}. " + f"Got {sorted(item)}" + ) + assert item["id"], f"an opportunity came back with no id: {item}" + assert isinstance(item["number"], str) and item["number"].strip(), ( + f"empty `number` — hit.number is gone from grants.gov's response: {item}" + ) + assert isinstance(item["title"], str) and item["title"].strip(), ( + f"empty `title` — hit.title is gone from grants.gov's response: {item}" + ) + + ids = [str(o["id"]) for o in listed] + assert len(set(ids)) == len(ids), ( + f"list_posted_opportunities returned {len(ids) - len(set(ids))} duplicate ids — " + "startRecordNum is not advancing, so every page is the same page" + ) + if hit_count > _PAGE_SIZE: + assert len(listed) > _PAGE_SIZE, ( + f"grants.gov reports {hit_count} hits but we collected {len(listed)} " + f"(= one page of {_PAGE_SIZE}) — pagination stopped after the first page" + ) + assert len(listed) >= hit_count * 0.9, ( + f"collected {len(listed)} of {hit_count} reported hits — the pager is dropping " + "pages (a little slack is allowed for the index changing mid-run)" + ) + + # The agency filter is a parameter we send; if it stopped being honoured we would + # be flooding GrantBot with every agency's postings and never notice. + off_target = sorted({o["agency"] for o in listed} - set(agencies)) + assert not off_target, ( + f"asked grants.gov for {agencies} and got {off_target} back — the `agencies` " + "payload field was ignored or is being joined with the wrong separator" + ) + + # Dates: `_parse_close_date` (agent/grantbot.py) returns None for anything it cannot + # read, and a None deadline is treated as "rolling" and PASSES the lead-time filter. + # A format change would therefore silently disable lead-time filtering entirely, + # which is exactly the kind of failure only a live test can see. + dated = [o for o in listed if o["close_date"]] + assert dated, ( + "no posted opportunity carried a close_date — hit.closeDate is gone, and " + "grantbot's lead-time filter would treat every FOA as rolling" + ) + unparsed = [o["close_date"] for o in dated if _parse_close_date(o["close_date"]) is None] + assert len(unparsed) <= len(dated) * 0.1, ( + f"{len(unparsed)} of {len(dated)} close_dates are unparseable by " + "src.agent.grantbot._parse_close_date, which accepts %m/%d/%Y, %Y-%m-%d and " + f"%Y/%m/%d — grants.gov changed its date format. Examples: {unparsed[:5]}" + ) + opened = [o for o in listed if o["open_date"]] + bad_open = [o["open_date"] for o in opened if _parse_close_date(o["open_date"]) is None] + assert len(bad_open) <= len(opened) * 0.1, ( + f"{len(bad_open)} of {len(opened)} open_dates are unparseable — grants.gov " + f"changed its date format. Examples: {bad_open[:5]}" + ) + + +# --------------------------------------------------------------------------- T3.2 + + +async def test_the_contract_search_fixture_still_matches_grants_gov(api_budget): + """Rule L1, the load-bearing test in this file. + + Walks every key path the hand-written search-hit literals in + tests/contract/test_grants_contract.py assert on and requires it to exist in a live + oppHits entry. + + Control: minimum path counts are asserted on both sides first. If the extractor + found nothing, or grants.gov returned an empty page, `renamed` would be trivially + empty and a pass would prove nothing — which is the precise failure mode Rule L1 is + about. + """ + fixtures = contract_fixture_literals()["search"] + assert len(fixtures) >= 3, ( + f"only extracted {len(fixtures)} search-hit literals from {_CONTRACT_FILE.name} " + "— the AST extractor is broken (or the contract file was restructured), so the " + "comparison below would be vacuous" + ) + fixture_paths = {p for f in fixtures for p in key_paths(f) if p} + assert len(fixture_paths) >= 6, ( + f"the fixture walker found only {len(fixture_paths)} paths: {sorted(fixture_paths)}" + ) + + api_budget.wait("grants") + raw = await _raw_post(grants.SEARCH_URL, { + "oppStatuses": "posted", + "agencies": "|".join(grants.BIOMEDICAL_AGENCIES), + "rows": 25, + "startRecordNum": 0, + }) + hits = raw.get("data", {}).get("oppHits") or [] + assert len(hits) >= 5, ( + f"grants.gov returned {len(hits)} oppHits for a broad posted search — provider " + "degraded or the envelope moved; the drift comparison would be meaningless" + ) + + # Union across hits: an optional key absent from one opportunity is not a rename. + live_empty: set[str] = set() + live_paths = {p for h in hits for p in key_paths(h, empty=live_empty) if p} + assert len(live_paths) >= 8, ( + f"a live oppHit has only {len(live_paths)} key paths ({sorted(live_paths)}) — " + "grants.gov's response shrank dramatically" + ) + + absent = [p for p in fixture_paths if p not in live_paths] + # A path under an EMPTY live container tells us nothing: the key may be intact and + # simply have no rows in these opportunities. + unverifiable = sorted(p for p in absent if any(p.startswith(e + ".") for e in live_empty)) + renamed = sorted(p for p in absent if p not in unverifiable) + + assert not renamed, ( + "grants.gov's live search2 oppHits no longer contain key paths that " + "tests/contract/test_grants_contract.py's hand-written fixtures assert on, and " + "their parent containers are NOT empty — this is a real schema difference and " + "those contract tests are pinning a shape grants.gov does not return:\n " + + "\n ".join(renamed) + + f"\nLive keys actually present: {sorted(live_paths)}" + ) + verified = fixture_paths - set(unverifiable) + assert len(verified) >= 5, ( + f"only {len(verified)} fixture paths could be checked against live data " + f"({len(unverifiable)} sit under empty containers: {unverifiable}) — this run " + "proved almost nothing" + ) + + +async def test_the_contract_detail_fixture_still_matches_grants_gov(api_budget): + """Rule L1 for fetchOpportunity. Same technique as the search drift test. + + Rule L3: when grants.gov's detail backend is unavailable this SKIPS with the + provider's own message rather than passing. A green run here must mean "verified", + never "could not look". + """ + fixtures = contract_fixture_literals()["detail"] + assert len(fixtures) >= 1, ( + f"extracted no detail literals from {_CONTRACT_FILE.name} — the AST extractor " + "is broken and this comparison would be vacuous" + ) + fixture_paths = {p for f in fixtures for p in key_paths(f) if p} + assert len(fixture_paths) >= 10, ( + f"the fixture walker found only {len(fixture_paths)} detail paths: " + f"{sorted(fixture_paths)}" + ) + + api_budget.wait("grants") + page = await grants.search_opportunities("cancer", agencies=["HHS-NIH11"], rows=5) + assert page, ( + "could not obtain any live opportunity to look up — search2 is down or its " + "oppHits path moved; the detail fixture is unchecked either way" + ) + opp_id = str(page[0]["id"]) + + api_budget.wait("grants") + raw = await _raw_post(grants.DETAIL_URL, {"oppId": opp_id}) + outage = detail_backend_outage(raw) + if outage: + pytest.skip( + "PROVIDER DOWN, not a schema change: grants.gov fetchOpportunity answered " + f"HTTP 200 / errorcode {raw.get('errorcode')!r} / msg {raw.get('msg')!r} but " + f"its backend reported {outage!r}. The detail-endpoint half of " + "test_grants_contract.py is therefore UNVERIFIED." + ) + + live_empty: set[str] = set() + live_paths = {p for p in key_paths(raw.get("data", {}), empty=live_empty) if p} + assert len(live_paths) >= 8, ( + f"the detail response has only {len(live_paths)} key paths — grants.gov " + "returned something unexpected and the comparison would be meaningless" + ) + absent = [p for p in fixture_paths if p not in live_paths] + unverifiable = sorted(p for p in absent if any(p.startswith(e + ".") for e in live_empty)) + renamed = sorted(p for p in absent if p not in unverifiable) + assert not renamed, ( + "grants.gov's fetchOpportunity response no longer contains key paths that " + "tests/contract/test_grants_contract.py asserts on, and their parents are not " + "empty — a real schema change:\n " + "\n ".join(renamed) + + f"\nLive keys actually present: {sorted(live_paths)}" + ) + verified = fixture_paths - set(unverifiable) + assert len(verified) >= 6, ( + f"only {len(verified)} of {len(fixture_paths)} detail fixture paths were " + f"checkable ({len(unverifiable)} under empty containers: {unverifiable})" + ) + + +# --------------------------------------------------------------------------- T3.3 + + +async def test_search_opportunities_narrows(api_budget): + """Two independent narrowings, because they catch two different mutations: dropping + the `keyword` field, and dropping/mis-joining the `agencies` field. + + Control: each narrow query must return >= 1. Without that, "fewer" is satisfied by a + search that is simply broken — the single most likely way this test would lie. + """ + rows = 200 # must exceed the narrow result counts or the cap decides the comparison + + api_budget.wait("grants") + broad = await grants.search_opportunities("research", rows=rows) + api_budget.wait("grants") + narrow = await grants.search_opportunities("cancer", rows=rows) + api_budget.wait("grants") + narrower = await grants.search_opportunities("cancer", agencies=["HHS-NIH11"], rows=rows) + + assert len(broad) >= 1, ( + "the broad query returned nothing at all — grants.gov is down, rate limiting " + "us, or search2's oppHits path moved. Nothing below can be concluded" + ) + assert len(narrow) >= 1, ( + "CONTROL FAILED: the narrow query ('cancer') returned nothing, so a smaller " + "result set would prove only that the search is broken, not that it narrows" + ) + assert len(narrow) < len(broad), ( + f"'cancer' returned {len(narrow)} and 'research' returned {len(broad)} — a more " + "specific keyword did not narrow the result set, so the `keyword` field is " + "probably not reaching grants.gov (or both queries hit the rows cap of " + f"{rows}, which would also make this comparison meaningless)" + ) + + assert len(narrower) >= 1, ( + "CONTROL FAILED: 'cancer' filtered to HHS-NIH11 returned nothing, so 'fewer " + "than unfiltered' proves nothing" + ) + assert len(narrower) < len(narrow), ( + f"filtering 'cancer' to HHS-NIH11 returned {len(narrower)}, the same or more " + f"than the unfiltered {len(narrow)} — the `agencies` payload field is being " + "ignored" + ) + off_target = sorted({o["agency"] for o in narrower} - {"HHS-NIH11"}) + assert not off_target, ( + f"asked for HHS-NIH11 only and got {off_target} — the agency filter is not " + "being applied (wrong payload key, or the '|' join changed)" + ) + for opp in narrower: + assert set(opp) == SEARCH_KEYS, ( + "search_opportunities' mapped keys changed — callers read " + f"{sorted(SEARCH_KEYS)}. Got {sorted(opp)}" + ) + + +# --------------------------------------------------------------------------- T3.4 + + +async def test_fetch_opportunity_detail_round_trips_an_id_from_the_live_page(api_budget): + """Rule L2: the id comes from the live page fetched moments earlier, so this test + can never go stale the way a pinned opp_id would. + + Rule L3, four outcomes: unreachable (raises), grants.gov's own backend down + (documented envelope -> skip), a real body that our parser threw away (fail, ours), + or the round trip (pass). + """ + api_budget.wait("grants") + page = await grants.search_opportunities("cancer", agencies=["HHS-NIH11"], rows=5) + assert page, ( + "search2 returned no opportunity to look up — provider down or the oppHits " + "path moved; the round trip is untested either way" + ) + source = page[0] + opp_id, opp_number = str(source["id"]), source["number"] + + api_budget.wait("grants") + try: + detail = await grants.fetch_opportunity_detail(opp_id) + except httpx.HTTPError as exc: + pytest.fail( + f"fetch_opportunity_detail({opp_id!r}) raised {exc!r} — grants.gov's detail " + "endpoint is unreachable or non-200. Note the caller (agent/tools.py) does " + "not catch this" + ) + + if detail is None: + api_budget.wait("grants") + raw = await _raw_post(grants.DETAIL_URL, {"oppId": opp_id}) + outage = detail_backend_outage(raw) + if outage: + # Graceful degradation still asserted: a valid id during an outage must + # yield None, not a half-populated dict the agents would post as fact. + assert detail is None + pytest.skip( + "PROVIDER DOWN, not our parser: grants.gov fetchOpportunity answered " + f"HTTP 200 / errorcode {raw.get('errorcode')!r} / msg {raw.get('msg')!r} " + f"for the live id {opp_id} but its backend reported {outage!r}. " + "fetch_opportunity_detail correctly returned None. The id round trip is " + "UNVERIFIED today." + ) + pytest.fail( + f"OUR PARSER: grants.gov returned a real body for oppId {opp_id} but " + "fetch_opportunity_detail returned None — its `data.get('number')` guard is " + f"discarding a valid opportunity. Live data keys: " + f"{sorted(raw.get('data', {}))}" + ) + + assert str(detail["id"]) == opp_id, ( + f"asked for oppId {opp_id} and got back id {detail['id']!r} — the detail " + "endpoint is answering with a different opportunity, which would attribute the " + "wrong FOA to a PI" + ) + assert detail["number"].upper() == opp_number.upper(), ( + f"id {opp_id} is number {opp_number!r} in search2 but {detail['number']!r} in " + "fetchOpportunity — the two endpoints disagree about the same opportunity" + ) + assert set(detail) >= LIST_KEYS | {"synopsis", "award_ceiling", "eligibility"}, ( + f"fetch_opportunity_detail's mapped keys changed; agent/tools.py and " + f"agent/foa_cache.py read them. Got {sorted(detail)}" + ) + + # Round trip the other way: number -> opportunity must reach the same id. + api_budget.wait("grants") + api_budget.wait("grants") # by_number searches, then fetches detail + by_number = await grants.fetch_opportunity_by_number(opp_number) + assert by_number is not None, ( + f"fetch_opportunity_by_number({opp_number!r}) found nothing for a number that " + "grants.gov returned seconds ago — its keyword search (rows=5) did not surface " + "the exact match" + ) + assert str(by_number["id"]) == opp_id, ( + f"number {opp_number!r} resolved to id {by_number['id']!r}, not {opp_id} — the " + "number->id lookup is matching the wrong opportunity" + ) + + +# --------------------------------------------------------------------------- T3.5 + + +async def test_an_unknown_opportunity_number_returns_none(api_budget): + """Absence assertion + its positive control, in that order of importance. + + Control: a number taken from the live page must resolve in the SAME test. Without + it, `None` for the bogus number is equally well explained by grants.gov being down, + which is the Rule L3 confusion this test exists to prevent. + """ + api_budget.wait("grants") + page = await grants.search_opportunities("cancer", agencies=["HHS-NIH11"], rows=5) + assert page, ( + "no live opportunity available — grants.gov is down, so the negative result " + "below would prove nothing" + ) + real_number = page[0]["number"] + + api_budget.wait("grants") + api_budget.wait("grants") # search, then (attempted) detail + found = await grants.fetch_opportunity_by_number(real_number) + assert found is not None, ( + f"CONTROL FAILED: {real_number!r} came from grants.gov seconds ago but " + "fetch_opportunity_by_number returned None for it. Until a real number " + "resolves, `None` for a fake one means nothing" + ) + assert found.get("number", "").upper() == real_number.upper(), ( + f"asked for {real_number!r}, got {found.get('number')!r} — the number match in " + "fetch_opportunity_by_number is returning a different opportunity" + ) + + api_budget.wait("grants") + try: + missing = await grants.fetch_opportunity_by_number(BOGUS_NUMBER) + except httpx.HTTPError as exc: + pytest.fail( + f"fetch_opportunity_by_number({BOGUS_NUMBER!r}) raised {exc!r} instead of " + "returning None — an unknown FOA number must degrade, not propagate a " + "transport error to the agent loop" + ) + assert missing is None, ( + f"a nonexistent FOA number resolved to {missing!r} — grants.gov's keyword search " + "is fuzzy, and the exact-number guard in fetch_opportunity_by_number is the only " + "thing stopping an agent from citing an unrelated opportunity" + ) + + +# --------------------------------------------------------------------------- T3.6 + + +async def test_search_for_researchers_matches_real_keywords_and_not_gibberish(api_budget): + """Positive and negative in one call, so "no results" can be attributed. + + Control: the real-keyword researcher must come back non-empty. `search_for_researchers` + swallows every exception per keyword, so an empty result for the gibberish researcher + is otherwise indistinguishable from grants.gov refusing the request entirely. + + The real keyword is listed twice on purpose: that guarantees the dedup path is + exercised, so the uniqueness assertion below cannot pass vacuously. + """ + keyword = "cancer immunotherapy" + query = {"real": [keyword, keyword], "gibberish": GIBBERISH} + + for _ in range(len(query["real"]) + len(query["gibberish"])): + api_budget.wait("grants") + out = await grants.search_for_researchers(query, max_per_query=5) + + assert set(out) == {"real", "gibberish"}, ( + f"search_for_researchers dropped a researcher from its result map: {sorted(out)}" + " — every agent_id must get a key even when nothing matched" + ) + real = out["real"] + assert real, ( + f"CONTROL FAILED: {keyword!r} matched no posted " + f"{grants.BIOMEDICAL_AGENCIES} opportunity. Either grants.gov is down/rate " + "limiting (search_for_researchers swallows the exception and logs a warning) or " + "the query is not reaching it. The gibberish result below proves nothing until " + "this passes" + ) + assert out["gibberish"] == [], ( + f"nonsense keywords {GIBBERISH} matched {len(out['gibberish'])} opportunities " + f"({[o['number'] for o in out['gibberish'][:5]]}) — the keyword is being ignored " + "and every researcher would be handed the same generic list" + ) + + assert len(real) >= 2, ( + f"only {len(real)} result(s) for {keyword!r}; the duplicate-keyword dedup " + "control needs at least two, so the uniqueness assertion below is weak" + ) + numbers = [o["number"] for o in real] + assert len(set(numbers)) == len(numbers), ( + f"the same keyword was searched twice and produced duplicate FOA numbers " + f"({len(numbers) - len(set(numbers))} of them) — the `seen_for_agent` dedup in " + "search_for_researchers is not working, and agents would see each opportunity " + "once per matching keyword" + ) + for opp in real: + assert set(opp) == SEARCH_KEYS | {"matched_keyword"}, ( + "search_for_researchers' result keys changed — grantbot.py reads them to " + f"build its prompt. Got {sorted(opp)}" + ) + assert opp["matched_keyword"] == keyword, ( + f"matched_keyword is {opp['matched_keyword']!r}, not the keyword that " + "produced the hit — the provenance tag GrantBot cites is wrong" + ) + assert opp["agency"] in grants.BIOMEDICAL_AGENCIES, ( + f"{opp['agency']!r} is outside the default agency filter " + f"{grants.BIOMEDICAL_AGENCIES} — search_for_researchers is not passing " + "`agencies` through to search_opportunities" + ) diff --git a/tests/live_api/test_marker.py b/tests/live_api/test_marker.py new file mode 100644 index 0000000..4215b4c --- /dev/null +++ b/tests/live_api/test_marker.py @@ -0,0 +1,24 @@ +"""Control for the live_api skip logic. + +Lives here, not in conftest.py: pytest does not collect tests from conftest, so the +first version of this control was never run — which is precisely the silent-disable +failure it exists to detect. +""" + +import os + +import pytest + + +@pytest.mark.live_api +def test_live_api_marker_actually_runs_when_configured(): + assert os.environ.get("LIVE_API_TESTS") == "1" + + +@pytest.mark.live_api +def test_api_budget_enforces_a_ceiling(api_budget): + """The budget must actually count. A no-op fixture would let a looping test hammer + a provider until the IP is throttled for everyone.""" + before = sum(api_budget.counts.values()) + api_budget.wait("orcid") + assert sum(api_budget.counts.values()) == before + 1 diff --git a/tests/live_api/test_orcid_live.py b/tests/live_api/test_orcid_live.py new file mode 100644 index 0000000..211b482 --- /dev/null +++ b/tests/live_api/test_orcid_live.py @@ -0,0 +1,173 @@ +"""ORCID Public API v3.0, live. + +ORCID is the highest-consequence external integration in the system: it is the login +path (`routers/auth.py`) as well as the profile source. + +Rule L1 is why `test_the_contract_fixture_still_matches_orcid` exists. +`tests/contract/test_orcid_contract.py` builds its record from a HAND-WRITTEN dict — +`_record()` returns a literal, not a recorded response. All 12 of those tests would +still pass if ORCID renamed a field tomorrow. Nothing checked that belief against +reality until this file. +""" + +import httpx +import pytest + +from src.services import orcid + +pytestmark = [pytest.mark.live_api] + +# ORCID maintains this as a permanent public example persona. It will not be deleted, +# made private, or renamed, which is what makes it safe to assert on (Rule L2). +CARBERRY = "0000-0002-1825-0097" + + +def key_paths(obj, prefix="", empty=None): + """Every dotted key path in a nested dict/list structure. + + ``empty`` collects the paths of containers that are present but EMPTY. That + distinction is the whole difference between "ORCID renamed a field" and "this + particular record has nothing in that list", and conflating them makes the drift + check cry wolf. Measured: the example record's person.emails.email is `[]`, so a + naive comparison reports person.emails.email[].email as missing. + """ + if isinstance(obj, dict): + if not obj: + if empty is not None: + empty.add(prefix) + yield prefix + return + for k, v in obj.items(): + yield from key_paths(v, f"{prefix}.{k}" if prefix else k, empty) + elif isinstance(obj, list): + if obj: + yield from key_paths(obj[0], f"{prefix}[]", empty) + else: + if empty is not None: + empty.add(f"{prefix}[]") + yield f"{prefix}[]" + else: + yield prefix + + +async def test_a_real_public_record_fetches_and_parses(api_budget): + """Shape first (Rule L2), then one dated value. + + Control: the parser must NOT return the same thing for a different id, or an + implementation that ignored its argument would pass. + """ + api_budget.wait("orcid") + prof = await orcid.fetch_orcid_profile(CARBERRY) + + assert prof["orcid"] == CARBERRY + assert isinstance(prof.get("name"), str) and prof["name"].strip() + assert prof["name"] != CARBERRY, ( + "fetch_orcid_profile fell back to the raw id, which is what it does when the " + "name block is missing — ORCID's person.name shape may have changed" + ) + # Dated assertion, allowed to be updated: as of 2026-07-30 this record is Carberry. + assert "Carberry" in prof["name"], f"as-of-2026-07-30 value changed: {prof['name']!r}" + + +async def test_fetch_orcid_record_returns_the_documented_top_level_shape(api_budget): + api_budget.wait("orcid") + rec = await orcid.fetch_orcid_record(CARBERRY) + assert isinstance(rec, dict) + for key in ("person", "activities-summary"): + assert key in rec, ( + f"ORCID's record no longer has a top-level {key!r} — " + f"fetch_orcid_profile reads it unconditionally. Got: {sorted(rec)[:12]}" + ) + + +async def test_the_contract_fixture_still_matches_orcid(api_budget): + """Rule L1, the load-bearing test in this file. + + Walks every key path the hand-written contract fixture asserts on and requires it to + exist in the live response. + + Control: a minimum path count is asserted first. If the walker broke or the live + record came back empty, `missing` would be trivially empty and this would prove + nothing — which is the exact failure mode the whole rule is about. + """ + from tests.contract.test_orcid_contract import _record + + api_budget.wait("orcid") + live = await orcid.fetch_orcid_record(CARBERRY) + + fixture_paths = {p for p in key_paths(_record()) if p} + assert len(fixture_paths) >= 8, ( + f"the fixture walker found only {len(fixture_paths)} paths — it is broken, so " + "the comparison below would be vacuous" + ) + live_empty: set[str] = set() + live_paths = {p for p in key_paths(live, empty=live_empty) if p} + assert len(live_paths) >= 20, ( + f"the live record has only {len(live_paths)} paths — ORCID returned something " + "unexpected and the comparison below would be meaningless" + ) + + absent = [p for p in fixture_paths if p not in live_paths] + # A path under an EMPTY live container tells us nothing: the key may be intact and + # simply have no rows in this record. + unverifiable = sorted( + p for p in absent if any(p.startswith(e + ".") for e in live_empty) + ) + renamed = sorted(p for p in absent if p not in unverifiable) + + assert not renamed, ( + "ORCID's live response no longer contains key paths that " + "tests/contract/test_orcid_contract.py's hand-written fixture asserts on, and " + "their parent containers are NOT empty — so this is a real schema change and " + "those contract tests are pinning a shape that no longer exists:\n " + + "\n ".join(renamed) + ) + # Control: if everything the fixture claims sits under an empty container, this test + # verified nothing and must say so rather than report a pass. + verified = fixture_paths - set(unverifiable) + assert len(verified) >= 6, ( + f"only {len(verified)} fixture paths could be checked against live data " + f"({len(unverifiable)} sit under empty containers: {unverifiable}). Pick a " + "richer record — this run proved almost nothing." + ) + + +async def test_fetch_orcid_works_returns_a_list_of_dicts(api_budget): + """Control: an ORCID with no works returns [], not an error — the profile pipeline + must still onboard a PI who has published nothing under this id.""" + api_budget.wait("orcid") + works = await orcid.fetch_orcid_works(CARBERRY) + assert isinstance(works, list) + if works: + assert all(isinstance(w, dict) for w in works) + else: + pytest.skip("the example record currently has no works to shape-check") + + +async def test_fetch_orcid_grants_returns_titles_or_empty(api_budget): + api_budget.wait("orcid") + grants = await orcid.fetch_orcid_grants(CARBERRY) + assert isinstance(grants, list) + assert all(isinstance(g, str) for g in grants) + + +async def test_an_unknown_orcid_degrades_without_taking_down_the_caller(api_budget): + """Rule L3: distinguish "ORCID said no such record" from "we could not reach ORCID". + + fetch_orcid_record raises for status; the grants/works helpers swallow and return + []. Both behaviours are deliberate and both are asserted, because the callers rely + on the difference. + """ + bogus = "0000-0000-0000-0000" + api_budget.wait("orcid") + with pytest.raises(httpx.HTTPStatusError) as ei: + await orcid.fetch_orcid_record(bogus) + assert ei.value.response.status_code in (400, 404), ( + f"ORCID answered {ei.value.response.status_code} for a nonexistent id — that is " + "neither the documented 404 nor a network failure" + ) + + api_budget.wait("orcid") + assert await orcid.fetch_orcid_grants(bogus) == [] + api_budget.wait("orcid") + assert await orcid.fetch_orcid_works(bogus) == [] diff --git a/tests/live_api/test_pubmed_live.py b/tests/live_api/test_pubmed_live.py new file mode 100644 index 0000000..3dc16cb --- /dev/null +++ b/tests/live_api/test_pubmed_live.py @@ -0,0 +1,756 @@ +"""NCBI E-utilities (PubMed / PMC / ID-converter), live. Task T2. + +`src/services/pubmed.py` is the largest external surface in the system and the one +whose failure mode is silent-and-wrong rather than loud: a mis-parsed ArticleId +attributes someone else's paper to a PI (issue #5) and nothing downstream notices. + +Rule L1 is why the drift tests exist. `tests/contract/test_pubmed_contract.py` pins the +parser against a HAND-WRITTEN `EFETCH_XML` literal — not a recorded response. If NCBI +renames an element or moves an id, all 10 of those tests still pass and production +breaks. Two things are therefore checked against live data here: the element paths the +fixture claims NCBI emits, and the parsed keys the fixture claims our parser produces. + +Rule L3: every assertion message below names which of the four diagnoses it observed — +provider down, rate limited, schema changed, or our parser broken. + +Records were chosen for permanence (Rule L2). Only immutable facts are pinned: a +published DOI, a publication year, an assigned PMCID. Titles, abstracts and +availability are asserted for shape only. +""" + +import inspect +import re +import xml.etree.ElementTree as ET + +import httpx +import pytest +import respx + +from src.services import pubmed + +pytestmark = [pytest.mark.live_api] + +EUTILS = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils" + +# Watson & Crick 1953, Nature. Seventy years old, in every textbook, and structurally +# minimal: no abstract, no PMC deposit, no reference list. It is the "thin record" case +# the parser must survive. DOI and year are immutable properties of a published paper. +WATSON_CRICK_PMID = "13054692" +WATSON_CRICK_DOI = "10.1038/171737a0" +WATSON_CRICK_YEAR = 1953 + +# Jinek et al. 2012, Science (the CRISPR-Cas9 programmable-nuclease paper). Chosen as +# the "rich record": abstract, ELocationID, an NIH author manuscript in PMC, and — the +# reason it is here rather than any other paper — a carrying 23 OTHER +# papers' PMCIDs. That is the live trap for the issue-#5 bug. +JINEK_PMID = "22745249" +JINEK_DOI = "10.1126/science.1225829" +JINEK_PMCID = "PMC6286148" +JINEK_YEAR = 2012 + +# Li et al. 2009, Bioinformatics — "The Sequence Alignment/Map format and SAMtools". +# Fully open access in PMC, and its full text has a section titled "2 METHODS". Note +# that "2 methods" is NOT in `_extract_methods_section`'s exact-title set, so this +# record exercises the substring fallback, which is the tier that actually fires in +# production (see the namespace note in test_extract_methods_uses_the_unnamespaced...). +SAMTOOLS_PMCID = "PMC2723002" + +# Bolger et al. 2014, Bioinformatics — "Trimmomatic". Also fully open access with a +# 60 kB body, but its sections are titled ALGORITHMS / IMPLEMENTATION / RESULTS: there +# is no methods-titled section at all. This is the negative control for T2.4 — a real, +# complete, full-text article that must yield None rather than "". +TRIMMOMATIC_PMCID = "PMC4103590" + + +# --------------------------------------------------------------------------- helpers + + +def element_paths(root: ET.Element, prefix: str = "") -> set[str]: + """Every slash-delimited element path in an XML tree. + + Used to compare the hand-written fixture XML's shape against the live response. + Unlike the ORCID drift walker there is no empty-container problem here (an XML + element either exists or it does not), but the *record*-level analogue is real: + a thin record legitimately lacks . That is handled by choosing a rich + record and by classifying every miss below rather than failing on it blindly. + """ + here = f"{prefix}/{root.tag}" + out = {here} + for child in root: + out |= element_paths(child, here) + return out + + +def methods_titled_sections(xml_text: str) -> list[str]: + """Titles of every whose own mentions "method", namespace or not. + + This is an INDEPENDENT probe of the live XML — deliberately not routed through + `_extract_methods_section` — so a failure can be attributed. If this finds a + methods section and the service returns None, the service is broken; if this finds + none, the article genuinely has none and the None is correct. + """ + try: + root = ET.fromstring(xml_text) + except ET.ParseError: + return [] + found = [] + for tag in ("{http://jats.nlm.nih.gov}sec", "sec"): + title_tag = "title" if tag == "sec" else "{http://jats.nlm.nih.gov}title" + for sec in root.findall(f".//{tag}"): + t = sec.find(title_tag) + if t is not None and t.text and "method" in t.text.lower(): + found.append(t.text.strip()) + return found + + +async def _efetch_pubmed_xml(api_budget, pmid: str) -> str: + api_budget.wait("ncbi") + resp = await pubmed._ncbi_get( + f"{EUTILS}/efetch.fcgi", + {"db": "pubmed", "id": pmid, "rettype": "xml", "retmode": "xml"}, + ) + return resp.text + + +async def _efetch_pmc_xml(api_budget, pmcid: str) -> str: + api_budget.wait("ncbi") + resp = await pubmed._ncbi_get( + f"{EUTILS}/efetch.fcgi", + { + "db": "pmc", + "id": pmcid.replace("PMC", ""), + "rettype": "xml", + "retmode": "xml", + }, + ) + return resp.text + + +# ------------------------------------------------------------------------- T2.0 + + +@respx.mock +async def test_ncbi_get_sends_the_required_tool_and_email_parameters(): + """T2.0 — NCBI *requires* `tool=` and `email=` on every E-utilities request. + + Respx-mocked rather than live, but it lives in this tier because the consequence of + getting it wrong is a live block: NCBI throttles, then blocks by IP, anonymous + clients that do not identify themselves. A blocked IP breaks the whole tier for + everyone afterwards, so this runs first. + + Control: an assertion whose passing condition is the *presence* of two parameters is + worthless if the reader cannot see any parameters at all. So `db` — a parameter the + caller definitely passes — is asserted first. If `db` is visible and `tool`/`email` + are not, the omission is real and not an artefact of how the query string is read. + """ + route = respx.get(f"{EUTILS}/esummary.fcgi").mock( + return_value=httpx.Response(200, json={"result": {"uids": []}}) + ) + await pubmed._ncbi_get(f"{EUTILS}/esummary.fcgi", {"db": "pubmed", "id": "1"}) + + assert route.called, ( + "respx recorded no request — _ncbi_get did not reach the URL under test, so " + "nothing below is meaningful (EUTILS_BASE may have moved)" + ) + params = route.calls.last.request.url.params + assert params.get("db") == "pubmed", ( + "the caller's own parameters are not visible in the recorded query string, so " + f"this test cannot observe what _ncbi_get sends. Saw: {dict(params)}" + ) + + absent = [k for k in ("tool", "email") if not params.get(k)] + assert not absent, ( + f"_ncbi_get omits {absent} from every NCBI request. This is not a schema change " + "and not a network problem — src/services/pubmed.py never adds them. NCBI's " + "E-utilities usage policy requires both on every request and throttles, then " + "blocks by IP, clients that omit them; the whole profile pipeline runs through " + f"this one function. Saw: {dict(params)}" + ) + + +# ------------------------------------------------------------------------- T2.1 + + +async def test_two_stable_pubmed_records_fetch_and_parse(api_budget): + """T2.1 — shape for everything, exact values only for the immutable ones. + + Both records are requested in a SINGLE efetch call, which makes the control free: + a parser that ignored its input, returned a constant, or kept only the last article + in the set would not produce two distinct records with their own correct DOIs. + """ + api_budget.wait("ncbi") + recs = await pubmed.fetch_pubmed_records([WATSON_CRICK_PMID, JINEK_PMID]) + + assert len(recs) == 2, ( + f"efetch returned {len(recs)} parsed records for 2 PMIDs. Either NCBI is " + "degraded/rate-limiting (an empty list means _fetch_pubmed_batch swallowed an " + "HTTP error — see the logged 'Failed to fetch PubMed batch'), or the " + "PubmedArticle element the parser iterates on has been renamed. " + f"Got: {[r.get('pmid') for r in recs]}" + ) + by_pmid = {r["pmid"]: r for r in recs} + assert set(by_pmid) == {WATSON_CRICK_PMID, JINEK_PMID}, ( + "the parsed PMIDs are not the ones requested — the parser is reading a PMID " + f"from the wrong element. Got {sorted(by_pmid)}" + ) + + for pmid, rec in by_pmid.items(): + assert isinstance(rec.get("title"), str) and rec["title"].strip(), ( + f"PMID {pmid} parsed with an empty title; ArticleTitle is present on every " + "PubMed record, so this is a parser or schema problem, not missing data" + ) + assert isinstance(rec.get("journal"), str) and rec["journal"].strip(), ( + f"PMID {pmid} parsed with no journal — Journal/Title may have moved" + ) + assert isinstance(rec.get("pub_types"), list) and rec["pub_types"], ( + f"PMID {pmid} parsed with no PublicationType" + ) + assert isinstance(rec.get("author_count"), int) and rec["author_count"] > 0, ( + f"PMID {pmid} parsed with {rec.get('author_count')} authors — the Author " + "element may have moved" + ) + + # Immutable: a published DOI and a publication year never change (Rule L2). + assert by_pmid[WATSON_CRICK_PMID]["doi"] == WATSON_CRICK_DOI + assert by_pmid[WATSON_CRICK_PMID]["year"] == WATSON_CRICK_YEAR + assert by_pmid[JINEK_PMID]["doi"] == JINEK_DOI + assert by_pmid[JINEK_PMID]["year"] == JINEK_YEAR + assert by_pmid[JINEK_PMID]["pmcid"] == JINEK_PMCID + + # The rich record carries an abstract; the 1953 one does not. Both are correct, and + # asserting the difference is what proves the abstract path is really being read + # rather than filled with a constant. + assert by_pmid[JINEK_PMID]["abstract"].strip(), ( + "the 2012 record parsed with an empty abstract — AbstractText may have moved" + ) + assert by_pmid[WATSON_CRICK_PMID]["abstract"] == "", ( + "the 1953 record has no <Abstract> in PubMed, so the parser must yield '' for " + "it. A non-empty value here means abstract text is leaking in from elsewhere: " + f"{by_pmid[WATSON_CRICK_PMID]['abstract'][:120]!r}" + ) + + +# ------------------------------------------------------------------------- T2.2 + + +async def test_the_contract_fixture_still_matches_the_real_efetch_xml(api_budget): + """T2.2, part 1 — Rule L1 for the XML the fixture claims NCBI returns. + + `EFETCH_XML` in tests/contract/test_pubmed_contract.py is a literal somebody typed. + Every element path it uses is a belief about NCBI's schema that nothing has ever + checked. This walks those paths and requires them in a live response. + + Control: the live XML must be non-trivial (>1000 chars) and must yield a large path + set first. If efetch returned an error stub, `missing` would be everything (loud) — + but if the walker itself broke, `missing` would be empty and the test would pass + while proving nothing. Both are guarded. + """ + from tests.contract import test_pubmed_contract as fixture_mod + + live_xml = await _efetch_pubmed_xml(api_budget, JINEK_PMID) + assert len(live_xml) > 1000, ( + f"efetch returned only {len(live_xml)} chars. That is an error stub or a " + "rate-limit page, not a record — the comparison below would be vacuous. " + f"Body starts: {live_xml[:200]!r}" + ) + + fixture_paths = element_paths(ET.fromstring(fixture_mod.EFETCH_XML)) + assert len(fixture_paths) >= 15, ( + f"the fixture walker found only {len(fixture_paths)} element paths — it is " + "broken, so the comparison below would be meaningless" + ) + live_paths = element_paths(ET.fromstring(live_xml)) + assert len(live_paths) >= 25, ( + f"the live record has only {len(live_paths)} element paths — NCBI returned " + "something unexpected and the comparison below would be meaningless" + ) + + missing = sorted(p for p in fixture_paths if p not in live_paths) + assert not missing, ( + "NCBI's live efetch response no longer contains element paths that the " + "hand-written EFETCH_XML fixture in tests/contract/test_pubmed_contract.py " + "asserts on. This is a SCHEMA CHANGE, not a network or parser fault: those " + "contract tests are pinning a document shape that no longer exists.\n " + + "\n ".join(missing) + ) + + # The parser branches on two attributes, not just on element names. A rename here + # is invisible to the path comparison above and silently empties every DOI/PMCID. + live_root = ET.fromstring(live_xml) + assert any( + el.get("IdType") for el in live_root.findall(".//ArticleId") + ), "ArticleId no longer carries an IdType attribute — every DOI and PMCID would be dropped" + assert any( + el.get("EIdType") for el in live_root.findall(".//ELocationID") + ), "ELocationID no longer carries an EIdType attribute — the DOI fallback is dead" + + +async def test_the_parser_produces_the_same_keys_on_live_xml_as_on_the_fixture(api_budget): + """T2.2, part 2 — Rule L1 for the parser's OUTPUT contract. + + The contract tests assert on nine keys of `_parse_pubmed_xml`'s output. Whether the + real parser still produces those nine keys from a real document has never been + checked. Runs the real parser over real XML and compares key sets. + + Control (the T1 empty-container lesson): a key can be absent for two entirely + different reasons — NCBI changed the document, or *this record* simply has no such + datum. Each absent key is therefore classified against an INDEPENDENT XPath probe of + the same live XML. "The probe found the data and the parser dropped it" is a parser + bug; "the probe found nothing either" is unverifiable, not a failure. A minimum + verified count then stops an all-unverifiable run reporting a pass. + + The baseline is read out of the contract test's SOURCE, not out of the parser's + output for the fixture. Measured: deriving it from `_parse_pubmed_xml(EFETCH_XML)` + makes the whole test vacuous, because a parser mutated to drop a key drops it from + both sides of the comparison and the drift check passes. (A mutant that deleted the + `pmcid` key survived this test until the baseline was moved off the parser.) + """ + from tests.contract import test_pubmed_contract as fixture_mod + + contract_test = fixture_mod.test_fetch_pubmed_records_parses_article_scoped_fields + fixture_keys = set(re.findall(r'\br\["(\w+)"\]', inspect.getsource(contract_test))) + assert len(fixture_keys) >= 8, ( + f"only {sorted(fixture_keys)} could be read out of the contract test's source. " + "The scraper is broken (or the test was rewritten), so the comparison below " + "would be vacuous" + ) + + live_xml = await _efetch_pubmed_xml(api_budget, JINEK_PMID) + assert len(live_xml) > 1000, ( + f"efetch returned {len(live_xml)} chars — an error stub or rate-limit page, " + "so nothing below would be meaningful" + ) + + fixture_recs = pubmed._parse_pubmed_xml(fixture_mod.EFETCH_XML) + assert len(fixture_recs) == 1, ( + "the parser no longer parses its own contract fixture — the drift comparison " + "has no baseline" + ) + missing_on_fixture = sorted(fixture_keys - set(fixture_recs[0])) + assert not missing_on_fixture, ( + "OUR PARSER no longer produces keys that tests/contract/test_pubmed_contract.py " + f"asserts on, even for that file's own hand-written XML: {missing_on_fixture}. " + "This is a parser regression, visible without any network access" + ) + + live_recs = pubmed._parse_pubmed_xml(live_xml) + assert len(live_recs) == 1, ( + f"the real parser produced {len(live_recs)} records from a real single-article " + "efetch response. OUR PARSER (or NCBI's PubmedArticle element) is broken" + ) + live_keys = set(live_recs[0]) + + # Independent evidence that each datum exists in the live document, found without + # going through the parser under test. + root = ET.fromstring(live_xml) + probes = { + "pmid": lambda: root.find(".//PMID") is not None, + "title": lambda: root.find(".//ArticleTitle") is not None, + "abstract": lambda: root.find(".//AbstractText") is not None, + "journal": lambda: root.find(".//Journal/Title") is not None, + "year": lambda: root.find(".//PubDate/Year") is not None, + "pub_types": lambda: root.find(".//PublicationType") is not None, + "author_count": lambda: root.find(".//Author") is not None, + "doi": lambda: any( + e.get("IdType") == "doi" for e in root.findall(".//ArticleId") + ) + or any(e.get("EIdType") == "doi" for e in root.findall(".//ELocationID")), + "pmcid": lambda: any( + e.get("IdType") == "pmc" for e in root.findall(".//ArticleId") + ), + } + unknown = sorted(k for k in fixture_keys if k not in probes) + assert not unknown, ( + f"the contract fixture now produces keys this drift test has no probe for: " + f"{unknown}. Add a probe — until then the comparison silently skips them" + ) + + absent = [k for k in fixture_keys if k not in live_keys] + parser_dropped = sorted(k for k in absent if probes[k]()) + not_in_this_record = sorted(k for k in absent if not probes[k]()) + + assert not parser_dropped, ( + "OUR PARSER is broken (not NCBI): for these keys the datum is demonstrably " + "present in the live XML — an independent XPath probe found it — and " + f"_parse_pubmed_xml did not emit the key: {parser_dropped}. " + f"Parsed keys were {sorted(live_keys)}" + ) + verified = fixture_keys - set(not_in_this_record) + assert len(verified) >= 8, ( + f"only {len(verified)} of {len(fixture_keys)} fixture keys could be checked " + f"against live data ({not_in_this_record} are absent from this record too). " + "Pick a richer PMID — this run proved almost nothing" + ) + + +async def test_the_parser_ignores_reference_list_article_ids_on_a_live_record(api_budget): + """The issue-#5 regression, against live data. + + A recursive `.//ArticleId` search also matches the <ReferenceList>, whose ids belong + to *cited* papers; the old code kept the last match and stamped publications with a + reference's DOI or PMCID. Contract tests cannot catch a regression here because the + hand-written fixture has no reference list — this is the only test in the system + that runs the parser over a document where the trap is actually set. + + Control: the trap must be armed. The test asserts the live record really does carry + other papers' PMCIDs before asserting that the parser picked the article's own; if + NCBI stops shipping reference lists, this reports "inconclusive", not "pass". + """ + live_xml = await _efetch_pubmed_xml(api_budget, JINEK_PMID) + root = ET.fromstring(live_xml) + + own_container = root.find(".//PubmedArticle/PubmedData/ArticleIdList") + assert own_container is not None, ( + "PubmedData/ArticleIdList is gone from the live response — the parser reads " + "this exact path, so every DOI and PMCID would silently disappear" + ) + own_pmcids = { + e.text for e in own_container.findall("ArticleId") if e.get("IdType") == "pmc" + } + all_pmcids = { + e.text for e in root.findall(".//ArticleId") if e.get("IdType") == "pmc" + } + foreign = all_pmcids - own_pmcids + assert len(foreign) >= 2, ( + f"this record now carries only {len(foreign)} reference-scoped PMCIDs, so the " + "trap this test exists to spring is no longer set and the assertion below " + "would pass for a parser that reads the reference list. Pick a PMID whose " + "PubMed record still has a <ReferenceList>" + ) + + recs = pubmed._parse_pubmed_xml(live_xml) + assert len(recs) == 1 + assert recs[0]["pmcid"] == JINEK_PMCID, ( + "OUR PARSER attributed the wrong PMCID to the article. A PMCID is immutable " + f"once assigned, so {recs[0].get('pmcid')!r} is not a data change — if it is " + f"one of {sorted(foreign)[:3]} the reference-list scoping fix has regressed" + ) + assert recs[0]["doi"] == JINEK_DOI, ( + f"OUR PARSER attributed DOI {recs[0].get('doi')!r} to a paper whose published " + f"(immutable) DOI is {JINEK_DOI} — the same reference-list regression" + ) + + +# ------------------------------------------------------------------------- T2.3 + + +async def test_id_conversion_round_trips_and_a_nonsense_doi_maps_to_nothing(api_budget): + """T2.3 — PMID → authoritative DOI → PMID → PMCID, asserted as a round trip. + + Nothing here is pinned to a value NCBI could legitimately change except the PMCID, + which is immutable once assigned. The round trip is self-checking: whatever DOI + PubMed reports for this PMID must resolve back to the same PMID. + + Control: a syntactically valid but nonexistent DOI must map to NOTHING. Mapping it + to *some* PMID is the failure that silently attributes a stranger's paper to a PI, + and without this leg "the converter returned a mapping" is satisfied by a converter + that returns a mapping for anything. + """ + api_budget.wait("ncbi") + auth = await pubmed.fetch_authoritative_dois([JINEK_PMID]) + assert JINEK_PMID in auth, ( + "esummary returned no DOI for a PMID that has one. Either NCBI is degraded " + "(fetch_authoritative_dois swallows the error and returns {}), or the " + f"articleids/idtype JSON shape changed. Got: {auth}" + ) + assert auth[JINEK_PMID].lower() == JINEK_DOI.lower(), ( + f"esummary reports {auth[JINEK_PMID]!r} as the DOI for PMID {JINEK_PMID}, but " + f"the published (immutable) DOI is {JINEK_DOI}. Either the esummary parser is " + "reading the wrong articleids entry, or the value now carries a URL prefix" + ) + + api_budget.wait("ncbi") + api_budget.wait("ncbi") # idconv, plus a possible esearch fallback + back = await pubmed.convert_dois_to_pmids([auth[JINEK_PMID]]) + assert back.get(auth[JINEK_PMID]) == JINEK_PMID, ( + f"the DOI PubMed itself reports for PMID {JINEK_PMID} does not round-trip back " + f"to it. Got {back!r} — the ID converter's record shape (doi/pmid keys) or the " + "esearch fallback's idlist path has changed" + ) + + api_budget.wait("ncbi") + pmcids = await pubmed.convert_pmids_to_pmcids([JINEK_PMID]) + assert pmcids.get(JINEK_PMID) == JINEK_PMCID, ( + f"PMID {JINEK_PMID} no longer maps to {JINEK_PMCID}. A PMCID is immutable once " + f"assigned, so this is the converter, not the data. Got: {pmcids!r}" + ) + + # Control leg. Well-formed, registrant prefix 10.9999 is not issued to anyone. + bogus = "10.9999/copi-live-test-no-such-doi-2f4a1c" + api_budget.wait("ncbi") + api_budget.wait("ncbi") # idconv, then the esearch fallback for the unresolved DOI + nothing = await pubmed.convert_dois_to_pmids([bogus]) + assert nothing == {}, ( + f"a nonexistent DOI resolved to {nothing!r}. This is the failure that attributes " + "someone else's paper to a PI: either the ID converter's error-record check " + "(status == 'error') no longer matches, or the esearch fallback is returning " + "unrelated hits for a term that matches nothing" + ) + + +# ------------------------------------------------------------------------- T2.4 + + +async def test_pmc_methods_extraction_on_real_open_access_articles(api_budget): + """T2.4 — `fetch_pmc_methods` / `_extract_methods_section` against live PMC. + + Three legs, because "returns a methods section" and "returns None" are only + meaningful together: + + 1. an OA article WITH a methods section -> a non-empty string + 2. an OA article WITHOUT one -> None (not "", which the caller + cannot distinguish from "empty") + 3. a PMC record with no full text at all -> None, swallowed, not an exception + + Each leg is attributed against an independent probe of the same XML, so a failure + says whether PMC changed the article or our extractor broke. + """ + # --- leg 1: has a methods section ------------------------------------------- + raw = await _efetch_pmc_xml(api_budget, SAMTOOLS_PMCID) + assert len(raw) > 1000, ( + f"PMC returned {len(raw)} chars for {SAMTOOLS_PMCID} — an error stub or a " + f"metadata-only record, not full text. Body starts: {raw[:200]!r}" + ) + titled = methods_titled_sections(raw) + assert titled, ( + f"{SAMTOOLS_PMCID}'s live full text no longer has any section titled with " + "'method'. That is a change in the TEST DATA, not a bug in the extractor — " + "pick another open-access article, because the leg below cannot pass" + ) + + methods = pubmed._extract_methods_section(raw) + assert isinstance(methods, str) and methods.strip(), ( + f"OUR EXTRACTOR returned {methods!r} even though an independent probe found " + f"methods-titled sections {titled} in the same XML. _extract_methods_section " + "is broken — most likely its <sec>/<title> traversal or its namespace handling" + ) + assert len(methods) > 200, ( + f"the extracted methods section is only {len(methods)} chars — the traversal " + f"is returning a title rather than the section body: {methods[:120]!r}" + ) + vocabulary = ("align", "sequence", "format", "algorithm", "index", "data") + hits = [w for w in vocabulary if w in methods.lower()] + assert hits, ( + "the extracted text contains none of the expected methodological vocabulary " + f"{vocabulary}, so the extractor probably grabbed the wrong section: " + f"{methods[:200]!r}" + ) + + # The public wrapper must agree with the extractor (it also strips the PMC prefix, + # which is the only transformation between them). + api_budget.wait("ncbi") + via_service = await pubmed.fetch_pmc_methods(SAMTOOLS_PMCID) + assert via_service is not None and via_service[:200] == methods[:200], ( + "fetch_pmc_methods disagrees with _extract_methods_section on the same article " + "— the wrapper's PMC-prefix stripping or its error swallowing is at fault, not " + f"the parser. Wrapper gave: {(via_service or '')[:120]!r}" + ) + + # --- leg 2: full text, but genuinely no methods section ---------------------- + raw_none = await _efetch_pmc_xml(api_budget, TRIMMOMATIC_PMCID) + assert len(raw_none) > 1000, ( + f"PMC returned {len(raw_none)} chars for {TRIMMOMATIC_PMCID}; this leg needs a " + "real full-text body, otherwise 'no methods section' is indistinguishable from " + "'no article'" + ) + assert not methods_titled_sections(raw_none), ( + f"{TRIMMOMATIC_PMCID} now HAS a methods-titled section, so it is no longer a " + "valid negative control. Change the TEST DATA, not the extractor" + ) + assert pubmed._extract_methods_section(raw_none) is None, ( + "OUR EXTRACTOR invented a methods section for an article that has none. The " + "caller distinguishes None from '' — returning either a string or '' here " + f"corrupts that: {pubmed._extract_methods_section(raw_none)!r}" + ) + + # --- leg 3: in PMC, but no full text deposited ------------------------------- + api_budget.wait("ncbi") + absent = await pubmed.fetch_pmc_methods(JINEK_PMCID) + assert absent is None, ( + f"{JINEK_PMCID} is a metadata-only PMC record (no <body>), so fetch_pmc_methods " + f"must return None. Got {type(absent).__name__} {str(absent)[:120]!r} — if this " + "is '' the caller can no longer tell 'no full text' from 'empty methods'" + ) + + +async def test_extract_methods_uses_the_unnamespaced_fallback_on_real_pmc_xml(api_budget): + """PMC's efetch output carries NO JATS namespace, so the first two tiers of + `_extract_methods_section` — both of which query `{http://jats.nlm.nih.gov}sec` — + never match a live response. Everything is done by the third, unnamespaced tier, + whose match is a loose `"method" in title` substring rather than the curated + exact-title set above it. + + This is asserted rather than left implicit because it means the exact-title + `methods_keywords` set is dead code against efetch, and anyone tightening the + substring tier would silently break every extraction. Control: the same document is + shown to contain sections in the unnamespaced form, so "the namespaced query found + nothing" cannot be explained by the document being empty. + """ + raw = await _efetch_pmc_xml(api_budget, SAMTOOLS_PMCID) + root = ET.fromstring(raw) + plain = root.findall(".//sec") + namespaced = root.findall(".//{http://jats.nlm.nih.gov}sec") + + assert plain, ( + "the live PMC document has no <sec> elements at all, so neither branch could " + "match and this test proves nothing — PMC's full-text shape has changed" + ) + assert not namespaced, ( + "PMC efetch now DOES emit JATS-namespaced <sec> elements. That is good news, " + "but it means _extract_methods_section's first two (exact-title) tiers have " + "started firing for the first time and their behaviour is now live — " + f"{len(namespaced)} namespaced sections found" + ) + + +# ------------------------------------------------------------------------- T2.5 + + +async def test_batching_covers_every_pmid_and_makes_the_expected_number_of_calls( + api_budget, monkeypatch +): + """T2.5 — `fetch_pubmed_records` chunks at 100; feed it more than one chunk. + + The ids come from a live esearch rather than a hard-coded list, so the test cannot + go stale and every id is guaranteed to exist right now (Rule L2). + + Two assertions with different targets: the CALL COUNT catches a batch size that + silently changed (one call for 120 ids would be an over-long URL NCBI rejects; 120 + calls would be a rate-limit ban), and the COVERAGE catches records dropped in the + middle of a multi-batch merge. Control for the call count: a single-chunk request + is also measured, so "calls == chunks" cannot be satisfied by a constant. + """ + want = 120 + api_budget.wait("ncbi") + resp = await pubmed._ncbi_get( + f"{EUTILS}/esearch.fcgi", + { + "db": "pubmed", + "term": 'crispr[tiab] AND 2018[dp] AND "journal article"[pt]', + "retmax": str(want), + "retmode": "json", + }, + ) + pmids = resp.json().get("esearchresult", {}).get("idlist", []) + assert len(pmids) == want, ( + f"esearch returned {len(pmids)} ids for retmax={want}; this test needs more " + "than one batch's worth or the batching maths below is untested" + ) + + original = pubmed._ncbi_get + calls: list[dict] = [] + + async def counting_ncbi_get(url, params): + calls.append(dict(params)) + api_budget.wait("ncbi") + return await original(url, params) + + monkeypatch.setattr(pubmed, "_ncbi_get", counting_ncbi_get) + + recs = await pubmed.fetch_pubmed_records(pmids) + assert len(calls) == 2, ( + f"{want} PMIDs produced {len(calls)} efetch calls; the code chunks at 100, so " + "2 is the only correct answer. 1 means the chunk size grew (NCBI rejects " + f"over-long id lists); {want} means it collapsed to one-per-id and this test " + "just spent 120 requests against a 3/s limit" + ) + sent = [len(p["id"].split(",")) for p in calls] + assert sent == [100, 20], f"batch sizes were {sent}, expected [100, 20]" + + returned = [r.get("pmid") for r in recs] + assert len(returned) == len(set(returned)), ( + "fetch_pubmed_records returned duplicate PMIDs — the multi-batch merge is " + "extending the result list with an earlier batch" + ) + foreign = sorted(set(returned) - set(pmids)) + assert not foreign, ( + f"records came back for PMIDs that were never requested: {foreign[:5]}. The " + "parser is reading a PMID from the wrong element (a CommentsCorrections or " + "reference entry), which is how a stranger's paper ends up on a PI's profile" + ) + dropped = sorted(set(pmids) - set(returned)) + assert not dropped, ( + f"{len(dropped)} of {want} requested PMIDs produced no parsed record: " + f"{dropped[:5]}. If the second batch is entirely missing, a batch's HTTP error " + "was swallowed by fetch_pubmed_records (check the log for 'Failed to fetch " + "PubMed batch' — that would be NCBI rate-limiting, not a parser fault); a " + "scattered few means those records are not PubmedArticle elements" + ) + + +# ------------------------------------------------------------------------- T2.6 + + +async def test_reconcile_pub_doi_separates_a_real_match_from_a_near_miss(api_budget): + """T2.6 — the gate that decides whether a paper is really this PI's. + + The authoritative DOI is taken live from esummary rather than hard-coded, so the + test exercises whatever format NCBI ships today: if esummary started returning + `https://doi.org/...` or a lower-cased variant, the "ok" leg below would fail and + every reconciled publication in production would be marked "corrected". + + Both directions are asserted in the same test. Without the "ok" leg, a function + that always answered "corrected" would pass; without the mismatch legs, one that + always answered "ok" would. + """ + api_budget.wait("ncbi") + auth_map = await pubmed.fetch_authoritative_dois([JINEK_PMID, WATSON_CRICK_PMID]) + for pmid in (JINEK_PMID, WATSON_CRICK_PMID): + assert pmid in auth_map, ( + f"esummary gave no authoritative DOI for PMID {pmid}; either NCBI is " + "degraded (the error is swallowed and {} returned) or articleids changed. " + f"Got {auth_map!r}" + ) + auth = auth_map[JINEK_PMID] + other = auth_map[WATSON_CRICK_PMID] + assert auth.lower() != other.lower(), "the two control DOIs must differ" + + # Permitted leg — an exact match. If this does not fire, every leg below is + # satisfied by a function that never matches anything. + assert pubmed.reconcile_pub_doi(auth, auth) == (auth, "ok"), ( + f"a DOI did not match itself. NCBI's esummary now reports {auth!r}, which " + "normalize_doi is not canonicalising to the stored form — in production every " + "correctly-attributed publication would be rewritten as 'corrected'" + ) + + # Same DOI, arriving in the two formats the ingest actually sees. + assert pubmed.reconcile_pub_doi(f"doi: {auth}", auth) == (auth, "ok"), ( + "a 'doi:'-prefixed DOI is no longer canonicalised to a match — normalize_doi's " + "prefix stripping regressed" + ) + assert pubmed.reconcile_pub_doi(f"https://doi.org/{auth}", auth) == (auth, "ok"), ( + "a doi.org URL is no longer canonicalised to a match — normalize_doi's URL " + "stripping regressed" + ) + assert pubmed.reconcile_pub_doi(auth.upper(), auth) == (auth.upper(), "ok"), ( + "DOIs are case-insensitive; an upper-cased assigned DOI must still match, and " + "the STORED form must be the one returned" + ) + + # Denied leg 1 — a whole different real paper's DOI on this PMID. This is exactly + # the issue-#5 corruption, and the gate must overwrite it with the authoritative one. + assert pubmed.reconcile_pub_doi(other, auth) == (auth, "corrected"), ( + f"the gate accepted {other!r} (a different, real paper) as the DOI for PMID " + f"{JINEK_PMID}, whose authoritative DOI is {auth!r}. This is the check that " + "stops someone else's paper being credited to a PI" + ) + + # Denied leg 2 — a near-miss: the same DOI with one character changed. Catches a + # comparison loosened to a prefix/substring match. + near = auth[:-1] + ("9" if auth[-1] != "9" else "8") + assert near.lower() != auth.lower() + assert pubmed.reconcile_pub_doi(near, auth) == (auth, "corrected"), ( + f"a one-character-off DOI ({near!r} vs {auth!r}) was accepted as a match — the " + "comparison has been loosened from equality to something fuzzier" + ) + + # The two remaining documented outcomes, so the action vocabulary is pinned whole. + assert pubmed.reconcile_pub_doi(None, auth) == (auth, "filled") + assert pubmed.reconcile_pub_doi(auth, None) == (auth, "unverified") + assert pubmed.reconcile_pub_doi(None, None) == (None, "none") + + # Sanity on the live value itself: a DOI is "10.<registrant>/<suffix>". + assert re.match(r"^10\.\d{4,9}/\S+$", auth), ( + f"esummary's authoritative DOI {auth!r} is not in DOI syntax — normalize_doi " + "is leaving a prefix on, or esummary changed its value format" + ) diff --git a/tests/unit/test_agent_prompts.py b/tests/unit/test_agent_prompts.py new file mode 100644 index 0000000..6dc2ee5 --- /dev/null +++ b/tests/unit/test_agent_prompts.py @@ -0,0 +1,105 @@ +from src.agent.agent import Agent +from src.agent.roles import DEFAULT_ROLE + + +def _agent(): + return Agent(agent_id="su", bot_name="SuBot", pi_name="Andrew Su") + + +def test_default_role_is_pi_lab(): + assert _agent().role == DEFAULT_ROLE + + +def test_identity_block_is_present_and_substituted(): + prompt = _agent().build_scan_system_prompt() + assert "You are **SuBot**" in prompt + assert 'the Andrew Su lab at Scripps Research' in prompt + assert 'agent ID is "su"' in prompt + + +def test_curly_brace_in_profile_does_not_crash(tmp_path, monkeypatch): + # A profile containing a bare "{" must not raise (str.replace, not str.format). + a = _agent() + monkeypatch.setattr(type(a), "public_profile", property(lambda self: "budget is {tight}")) + prompt = a.build_scan_system_prompt() # must not raise + assert "budget is {tight}" in prompt + + +def test_scan_prompt_omits_memory_and_lab_directory(): + a = _agent() + a._lab_directory = "### Other Lab\n- paper" + scan = a.build_scan_system_prompt() + assert "Other Lab" not in scan # scan prompt excludes the directory + + +def test_phase2_scan_prune_and_phase4_honour_role_overrides(tmp_path, monkeypatch): + """build_phase2_scan_prompt, build_phase2_prune_prompt, and build_phase4_prompt each + load their template via a hardcoded global path rather than the role-aware + resolver, so a role's override file for any of the three would be accepted into + the repo and then silently ignored. Pin that each one now resolves through + Agent._load_prompt (and therefore src.agent.roles.resolve_prompt_path).""" + from src.agent import roles as roles_mod + from src.agent.agent import Agent + from src.agent.state import PostRef, ThreadState + + prompts = tmp_path / "prompts" + (prompts / "roles" / "widget").mkdir(parents=True) + (prompts / "phase2-scan-filter.md").write_text("GLOBAL SCAN {posts}", encoding="utf-8") + (prompts / "phase2-prune.md").write_text("GLOBAL PRUNE {interesting_posts}", encoding="utf-8") + (prompts / "phase4-thread-reply.md").write_text("GLOBAL REPLY", encoding="utf-8") + (prompts / "roles" / "widget" / "phase2-scan-filter.md").write_text( + "WIDGET SCAN {posts}", encoding="utf-8" + ) + (prompts / "roles" / "widget" / "phase2-prune.md").write_text( + "WIDGET PRUNE {interesting_posts}", encoding="utf-8" + ) + (prompts / "roles" / "widget" / "phase4-thread-reply.md").write_text( + "WIDGET REPLY", encoding="utf-8" + ) + monkeypatch.setattr(roles_mod, "PROMPTS_DIR", prompts) + monkeypatch.setattr(roles_mod, "ROLES_DIR", prompts / "roles") + + agent = Agent("w", "WBot", "W Lab", role="widget") + _, scan_messages = agent.build_phase2_scan_prompt( + [{"post_id": "p1", "channel": "general", "sender": "x", "content_snippet": "s"}] + ) + assert "WIDGET SCAN" in scan_messages[0]["content"] + + agent.state.interesting_posts = [ + PostRef(post_id="p1", channel="general", sender_agent_id="x", content_snippet="s", posted_at=0.0) + ] + _, prune_messages = agent.build_phase2_prune_prompt() + assert "WIDGET PRUNE" in prune_messages[0]["content"] + + thread = ThreadState(thread_id="t1", channel="general", other_agent_id="o", message_count=1) + _, reply_messages = agent.build_phase4_prompt( + thread=thread, + thread_history=[{"sender": "o", "content": "hello"}], + other_agent_name="OBot", + other_agent_lab="O Lab", + ) + assert "WIDGET REPLY" in reply_messages[0]["content"] + + +def test_phase5_menu_token_is_always_substituted(): + """No caller may leak the raw token into a prompt. prompts/ is bind-mounted + and re-read per call while src/ is baked into the agent image, so a template + that ships ahead of its renderer would put `{post_type_menu}` in front of a + live model.""" + from src.agent.agent import Agent + + a = Agent("gill", "GillBot", "Gill") + _, messages = a.build_phase5_prompt() + assert "{post_type_menu}" not in messages[0]["content"] + + +def test_phase5_default_menu_never_prints_an_empty_enumeration(): + """The default path has no roster to enumerate from. Guarded here as well as + in test_post_types because this is the caller that reaches a snapshot.""" + from src.agent.agent import Agent + + a = Agent("gill", "GillBot", "Gill") + _, messages = a.build_phase5_prompt() + assert "one of: ." not in messages[0]["content"] + + diff --git a/tests/unit/test_cohort_isolation.py b/tests/unit/test_cohort_isolation.py new file mode 100644 index 0000000..3b41f7f --- /dev/null +++ b/tests/unit/test_cohort_isolation.py @@ -0,0 +1,1325 @@ +"""Cohort interaction gate + reactive-priority scheduler. + +Implements the test plan in .notes/cohort-system-v2.md §15. Organised by spec +section so a failure names the rule it broke: + +- TestGateHelper §5.1 the per-entry decision table +- TestComputeGates §5.2 policy semantics, shared by engine and admin +- TestPreflight §5.3 refusing to silence a roster +- TestGatedReads §6 MessageLog read filtering +- TestReadPathInventory §6 every public read method is classified +- TestStatePruning §6.1 stale interesting_posts +- TestDbPrimaryPaths §6.2 ingestion is never gated; is_bot keying +- TestPrivateChannels §7 PI pairings outrank the gate +- TestGrandfathering §8 resumed runs, conclude-but-deprioritise +- TestTagHygiene §9 outbound mention stripping +- TestScheduler §10 eligibility, fairness valve, ratio counters +- TestTopologySnapshot §13.1 provenance +- TestMigrationHygiene §14 single head, no duplicate revision ids +""" + +import inspect +import pathlib +import re +import types +import uuid + +import pytest + +from src.agent.agent import Agent +from src.agent.message_log import LogEntry, MessageLog, _entry_allowed +from src.agent.simulation import SimulationEngine +from src.agent.state import PostRef, ThreadState +from src.services.cohorts import ( + POLICY_ISOLATED, + POLICY_OPEN, + compute_gates, + preflight_reason, + summarise_gates, +) +from src.visibility import VISIBILITY_COLLAB_PRIVATE, VISIBILITY_PUBLIC + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _post( + ts, channel, agent_id, name, content, + thread_ts=None, is_bot=True, visibility=VISIBILITY_PUBLIC, +): + return LogEntry( + ts=ts, + channel=channel, + sender_agent_id=agent_id, + sender_name=name, + content=content, + thread_ts=thread_ts, + posted_at=float(ts), + is_bot=is_bot, + visibility=visibility, + ) + + +class _FakeResult: + """Mimics the slice of sqlalchemy Result the engine actually calls.""" + + def __init__(self, rows, scalar=None): + self._rows = rows + self._scalar = scalar + + def all(self): + return self._rows + + def scalar(self): + return self._scalar + + +class _FakeDB: + """Returns membership rows for the membership select, a count for the count.""" + + def __init__(self, rows, cohort_count=None): + self._rows = rows + self._cohort_count = ( + cohort_count if cohort_count is not None + else len({c for c, _ in rows}) + ) + self.added: list = [] + self.committed = False + + async def execute(self, stmt): + text = str(stmt).lower() + if "count" in text: + return _FakeResult([], scalar=self._cohort_count) + return _FakeResult(self._rows) + + def add(self, obj): + self.added.append(obj) + + async def commit(self): + self.committed = True + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + +def _engine(agent_ids, membership_rows=None, budget_cap=0, cohort_count=None): + agents = [ + Agent(agent_id=a, bot_name=f"{a.capitalize()}Bot", pi_name=f"PI {a}") + for a in agent_ids + ] + db = _FakeDB(membership_rows or [], cohort_count=cohort_count) + factory = (lambda: db) if membership_rows is not None else None + eng = SimulationEngine( + agents=agents, slack_clients={}, budget_cap=budget_cap, session_factory=factory + ) + name_map = {f"{a}bot": a for a in agent_ids} + eng.message_log.set_bot_name_map(name_map) + eng._bot_name_to_id = dict(name_map) + eng._fake_db = db # test handle + return eng + + +def _settings(**kw): + base = dict( + cohort_isolation_enabled=False, + cohort_default_policy=POLICY_OPEN, + max_consecutive_reactive_turns=3, + turn_delay_seconds=0.0, + # Required since _turn_eligible gained the rate limiter: _agent_load reads + # active_thread_threshold, _within_rate_limit reads the other two. Values are + # inert for this file — no test here opens a thread or records a call, so load + # is always 1 and the allowance is never approached. + active_thread_threshold=12, + llm_rate_window_seconds=600, + llm_calls_per_load_per_window=8, + ) + base.update(kw) + return types.SimpleNamespace(**base) + + +def _patch(monkeypatch, **kw): + monkeypatch.setattr( + "src.agent.simulation.get_settings", lambda: _settings(**kw) + ) + + +def _thread(agent, thread_id, other, pending=False, channel="general", grandfathered=False): + agent.state.active_threads[thread_id] = ThreadState( + thread_id=thread_id, channel=channel, other_agent_id=other, + has_pending_reply=pending, grandfathered=grandfathered, + ) + return agent.state.active_threads[thread_id] + + +# --------------------------------------------------------------------------- +# §5.1 — the per-entry decision table +# --------------------------------------------------------------------------- + + +class TestGateHelper: + def test_gate_off_passes_everything(self): + assert _entry_allowed(_post("1", "c", "z", "ZBot", "hi"), None) is True + + def test_human_always_passes(self): + e = _post("1", "c", None, "Dr PI", "hello", is_bot=False) + assert _entry_allowed(e, set()) is True + assert _entry_allowed(e, {"su"}) is True + + def test_human_with_agent_id_still_passes(self): + """is_bot is the human signal, not a NULL agent_id.""" + e = _post("1", "c", "su", "Dr PI", "hello", is_bot=False) + assert _entry_allowed(e, set()) is True + + def test_bot_with_null_agent_id_fails_closed(self): + """agent_messages.agent_id is nullable; an unattributable BOT row must not + slip through the human bypass. This is the hole that keying on + `sender_agent_id is None` opened.""" + e = _post("1", "c", None, "bot", "hi", is_bot=True) + assert _entry_allowed(e, {"su"}) is False + assert _entry_allowed(e, set()) is False + + def test_private_channel_always_passes(self): + e = _post("1", "priv", "cravatt", "CravattBot", "hi", + visibility=VISIBILITY_COLLAB_PRIVATE) + assert _entry_allowed(e, set()) is True + assert _entry_allowed(e, {"su"}) is True + + def test_cohort_mate_passes_non_mate_does_not(self): + assert _entry_allowed(_post("1", "c", "su", "SuBot", "hi"), {"su"}) is True + assert _entry_allowed(_post("1", "c", "z", "ZBot", "hi"), {"su"}) is False + + def test_empty_set_blocks_all_bots(self): + assert _entry_allowed(_post("1", "c", "su", "SuBot", "hi"), set()) is False + + +# The whole of §5.1 as data, so a failure names the row rather than the assertion, and +# so the table itself can be checked for polarity (see the control below). +DECISION_TABLE = [ + # (row name, _post kwargs, gate, expected_visible) + ("gate off", dict(agent_id="z"), None, True), + ("human", dict(agent_id=None, is_bot=False), set(), True), + ("human with agent_id", dict(agent_id="su", is_bot=False), set(), True), + ("private channel", dict(agent_id="z", + visibility=VISIBILITY_COLLAB_PRIVATE), set(), True), + ("cohort mate", dict(agent_id="su"), {"su"}, True), + ("non-mate", dict(agent_id="z"), {"su"}, False), + ("empty gate blocks bot", dict(agent_id="su"), set(), False), + ("bot, NULL agent_id", dict(agent_id=None, is_bot=True), {"su"}, False), + ("bot, empty agent_id", dict(agent_id="", is_bot=True), {"su"}, False), + ("unknown visibility", dict(agent_id="z", visibility="other"), {"su"}, False), +] + + +@pytest.mark.parametrize( + "name,kwargs,gate,expected", DECISION_TABLE, ids=[r[0] for r in DECISION_TABLE] +) +def test_decision_table_row(name, kwargs, gate, expected): + """Every §5.1 row, asserted in the direction the table states.""" + base = dict(ts="1", channel="c", agent_id="x", name="X", content="") + base.update(kwargs) + entry = _post(**base) + assert _entry_allowed(entry, gate) is expected, name + + +def test_decision_table_has_both_polarities(): + """Control for the table itself. + + A table of all-True rows would pass against a gate that never filters; a table of + all-False rows against one that filters everything. Neither would be noticed by the + parametrised test above, which is why this exists. + """ + outcomes = {row[3] for row in DECISION_TABLE} + assert outcomes == {True, False}, ( + f"the decision table must exercise both polarities, got {outcomes}" + ) + + +# --------------------------------------------------------------------------- +# §5.2 — policy semantics (the rule v1 documented and the code inverted) +# --------------------------------------------------------------------------- + + +class TestComputeGates: + def test_isolation_disabled_gates_are_none(self): + gates, reason = compute_gates( + membership_rows=[], agent_ids=["su", "wiseman"], + isolation_enabled=False, policy=POLICY_OPEN, cohort_count=0, + ) + assert reason is None + assert gates == {"su": None, "wiseman": None} + + def test_open_policy_zero_cohorts_is_a_no_op(self): + """The contract v1 published: enabling isolation with no cohorts defined + behaves exactly like all-vs-all.""" + gates, reason = compute_gates( + membership_rows=[], agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=0, + ) + assert reason is None + assert all(g is None for g in gates.values()) + + def test_open_policy_uncohorted_agent_is_unrestricted(self): + """Under `open`, unrestricted has to mean both directions. + + This assertion originally read `gates["su"] == {"su", "wiseman"}`, which pinned + a bug: it left the uncohorted agent reachable by nobody, so it could react and + never be replied to. See test_open_policy_is_symmetric_with_uncohorted_agents. + """ + c1 = uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=1, + ) + assert gates["su"] == {"su", "wiseman", "cravatt"} + assert gates["wiseman"] == {"su", "wiseman", "cravatt"} + assert gates["cravatt"] is None, "uncohorted agent must not be silenced" + + def test_open_policy_is_symmetric_with_uncohorted_agents(self): + """Regression: under policy=open an uncohorted agent must be reachable, not + merely able to reach. + + Its own gate is None so it may act on anyone; but a cohorted agent's gate is + the union of its co-members, which would not contain it. The result was an + agent that could react and never be replied to — it could not hold a + conversation, which is the opposite of "unrestricted", and it contradicts the + §5.1 row "`A` has no cohort memberships, policy = open -> Yes". + + Found by a real multi-turn run: the uncohorted agent opened two threads and no + cohorted agent ever replied. Every gate-computation test passed, and the + symmetry test skipped the case because it only compared pairs where both gates + were sets. + """ + c1 = uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=1, + ) + assert gates["cravatt"] is None, "the uncohorted agent stays unrestricted" + assert "cravatt" in gates["su"], ( + "a cohorted agent must be able to act on an uncohorted one under " + f"policy=open, else they can never converse. su gate={gates['su']}" + ) + assert "cravatt" in gates["wiseman"] + + def test_isolated_policy_does_not_add_uncohorted_agents(self): + """The fix must not leak into policy=isolated, where uncohorted means excluded.""" + c1 = uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert gates["su"] == {"su", "wiseman"} + assert "cravatt" not in gates["su"] + assert gates["cravatt"] == set() + + def test_isolated_policy_uncohorted_agent_gets_empty_set(self): + c1 = uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert gates["cravatt"] == set() + + def test_multi_cohort_union(self): + c1, c2 = uuid.uuid4(), uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "su"), (c1, "wiseman"), (c2, "su"), (c2, "cravatt")], + agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=2, + ) + assert gates["su"] == {"su", "wiseman", "cravatt"} + assert gates["wiseman"] == {"su", "wiseman"} + assert gates["cravatt"] == {"su", "cravatt"} + + def test_membership_for_offline_agent_is_inert(self): + """A membership naming an agent the engine isn't running must not appear in + anyone's mate set as a live sender, but must not break the computation.""" + c1 = uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "su"), (c1, "ghost")], + agent_ids=["su"], + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=1, + ) + assert "ghost" not in gates + # 'ghost' is still a co-member of the cohort, so su may act on it if it + # ever comes online — the roster, not the gate, decides who is running. + assert gates["su"] == {"su", "ghost"} + + def test_relation_is_symmetric(self): + c1 = uuid.uuid4() + gates, _ = compute_gates( + membership_rows=[(c1, "a"), (c1, "b")], agent_ids=["a", "b"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert ("b" in gates["a"]) and ("a" in gates["b"]) + + def test_summarise_gates(self): + s = summarise_gates({"a": None, "b": set(), "c": {"c", "d"}}) + assert s["total"] == 3 + assert s["gated"] == 2 + assert s["isolated"] == ["b"] + assert s["unrestricted"] == ["a"] + + +# --------------------------------------------------------------------------- +# §5.3 — preflight: never silently silence a roster +# --------------------------------------------------------------------------- + + +PREFLIGHT_CASES = [ + # (name, isolation, policy, cohort_count, has_db, live_members, refuses) + ("disabled always fine", False, POLICY_ISOLATED, 0, False, 0, False), + ("disabled, no db, isolated", False, POLICY_ISOLATED, 0, False, 0, False), + ("no db", True, POLICY_OPEN, 3, False, 3, True), + ("no db, isolated", True, POLICY_ISOLATED, 3, False, 3, True), + ("isolated, zero cohorts", True, POLICY_ISOLATED, 0, True, 0, True), + ("isolated, empty cohort", True, POLICY_ISOLATED, 1, True, 0, True), + ("isolated, offline member", True, POLICY_ISOLATED, 1, True, 0, True), + ("isolated, one member", True, POLICY_ISOLATED, 1, True, 1, False), + ("isolated, many members", True, POLICY_ISOLATED, 4, True, 9, False), + ("open, zero cohorts", True, POLICY_OPEN, 0, True, 0, False), + ("open, empty cohort", True, POLICY_OPEN, 1, True, 0, False), + ("open, members", True, POLICY_OPEN, 2, True, 5, False), +] + + +@pytest.mark.parametrize( + "name,iso,policy,n_cohorts,has_db,live,refuses", + PREFLIGHT_CASES, ids=[c[0] for c in PREFLIGHT_CASES], +) +def test_preflight_matrix(name, iso, policy, n_cohorts, has_db, live, refuses): + """Every combination of the four inputs preflight actually branches on.""" + reason = preflight_reason( + isolation_enabled=iso, policy=policy, cohort_count=n_cohorts, + has_db=has_db, live_members=live, + ) + assert (reason is not None) is refuses, f"{name}: reason={reason!r}" + + +def test_preflight_matrix_has_both_polarities(): + """Control: the matrix must contain refusing AND allowing rows. A matrix of one + polarity would pass against a preflight that always (or never) refused.""" + assert {c[6] for c in PREFLIGHT_CASES} == {True, False} + + +def test_preflight_allows_when_a_live_member_exists(): + """Positive control for the refusal cases. + + The refusals above are all assertions that isolation gets forced OFF. On their own + they would also be satisfied by a preflight that refused unconditionally. This shows + the same shape of input with one live member produces no refusal AND a gate that + actually filters. + """ + c1 = uuid.uuid4() + gates, reason = compute_gates( + membership_rows=[(c1, "su")], agent_ids=["su", "wiseman"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert reason is None + assert gates["su"] == {"su"} + assert gates["wiseman"] == set(), "isolation is in force, not forced off" + + +def test_open_policy_never_emits_an_empty_gate(): + """§5.4: under `open`, an empty set is a bug — it would silence the agent. + + Swept over every topology shape rather than one example, because the shapes differ + in which branch of compute_gates they take (uncohorted, solo, overlapping, offline + member). + """ + c1, c2 = uuid.uuid4(), uuid.uuid4() + shapes = [ + [], + [(c1, "su")], + [(c1, "su"), (c1, "wiseman")], + [(c1, "su"), (c2, "wiseman")], + [(c1, "su"), (c1, "wiseman"), (c2, "su"), (c2, "cravatt")], + [(c1, "ghost")], # member not on the roster + ] + for rows in shapes: + gates, reason = compute_gates( + membership_rows=rows, agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_OPEN, + cohort_count=len({r[0] for r in rows}), + ) + assert reason is None, rows + for aid, g in gates.items(): + assert g is None or g, f"{rows} produced an empty gate for {aid}" + + +def test_isolated_policy_does_emit_empty_gates(): + """Control for the test above: `isolated` is the policy where an empty gate is the + intended outcome. Without this, a compute_gates that never returned an empty set + would satisfy the no-empty-gate invariant for the wrong reason.""" + c1 = uuid.uuid4() + gates, reason = compute_gates( + membership_rows=[(c1, "su")], agent_ids=["su", "wiseman", "cravatt"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert reason is None + assert gates["wiseman"] == set() and gates["cravatt"] == set() + + +class TestPreflight: + def test_isolated_policy_with_zero_cohorts_is_refused(self): + reason = preflight_reason( + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=0, has_db=True + ) + assert reason and "roster-wide silence" in reason + + def test_open_policy_with_zero_cohorts_is_fine(self): + assert preflight_reason( + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=0, has_db=True + ) is None + + def test_isolated_policy_with_a_cohort_is_fine(self): + assert preflight_reason( + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + has_db=True, live_members=2, + ) is None + + def test_isolated_policy_with_an_empty_cohort_is_refused(self): + """Regression: the check must count live members, not cohorts. Creating a + cohort and never adding anyone to it silences the whole roster just as + surely as defining no cohorts at all.""" + reason = preflight_reason( + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + has_db=True, live_members=0, + ) + assert reason and "no live agent is a member" in reason + + def test_compute_gates_refuses_an_empty_cohort_under_isolated_policy(self): + gates, reason = compute_gates( + membership_rows=[], agent_ids=["su", "wiseman"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert reason is not None + assert all(g is None for g in gates.values()) + + def test_compute_gates_refuses_when_only_offline_agents_are_members(self): + """A cohort containing only agents the engine isn't running leaves every + live agent uncohorted.""" + c1 = uuid.uuid4() + gates, reason = compute_gates( + membership_rows=[(c1, "ghost")], agent_ids=["su"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert reason is not None and gates["su"] is None + + def test_one_live_member_is_enough_to_proceed(self): + c1 = uuid.uuid4() + gates, reason = compute_gates( + membership_rows=[(c1, "su")], agent_ids=["su", "wiseman"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=1, + ) + assert reason is None + assert gates["su"] == {"su"} and gates["wiseman"] == set() + + def test_no_database_is_refused(self): + reason = preflight_reason( + isolation_enabled=True, policy=POLICY_OPEN, cohort_count=3, has_db=False + ) + assert reason and "silently do nothing" in reason + + def test_disabled_isolation_never_refuses(self): + assert preflight_reason( + isolation_enabled=False, policy=POLICY_ISOLATED, cohort_count=0, has_db=False + ) is None + + def test_refusal_forces_every_gate_open(self): + gates, reason = compute_gates( + membership_rows=[], agent_ids=["su", "wiseman"], + isolation_enabled=True, policy=POLICY_ISOLATED, cohort_count=0, + ) + assert reason is not None + assert all(g is None for g in gates.values()), "must fail OPEN, not closed" + + async def test_engine_logs_error_and_disables(self, monkeypatch, caplog): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + eng = _engine(["su", "wiseman"], membership_rows=[], cohort_count=0) + with caplog.at_level("ERROR"): + await eng._recompute_allowed_sender_ids() + assert eng._cohort_preflight_error is not None + assert all(a.allowed_sender_ids is None for a in eng.agents.values()) + assert any("forced OFF" in r.getMessage() for r in caplog.records) + + async def test_engine_without_session_factory_disables(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True) + eng = _engine(["su", "wiseman"]) # membership_rows=None -> no factory + await eng._recompute_allowed_sender_ids() + assert eng._cohort_preflight_error is not None + assert all(a.allowed_sender_ids is None for a in eng.agents.values()) + + async def test_transient_db_error_leaves_gates_in_place(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman"], membership_rows=[(c1, "su"), (c1, "wiseman")]) + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su", "wiseman"} + + class _Boom: + async def execute(self, _s): + raise RuntimeError("connection reset") + + async def __aenter__(self): + return self + + async def __aexit__(self, *e): + return False + + eng.session_factory = lambda: _Boom() + await eng._recompute_allowed_sender_ids() + assert eng.agents["su"].allowed_sender_ids == {"su", "wiseman"}, ( + "a DB blip must not flap the gate open" + ) + + +# --------------------------------------------------------------------------- +# §6 — gated reads +# --------------------------------------------------------------------------- + + +class TestGatedReads: + @pytest.fixture + def log(self): + ml = MessageLog() + ml.set_bot_name_map({"subot": "su", "wisemanbot": "wiseman", "cravattbot": "cravatt"}) + return ml + + def test_top_level_posts_filtered(self, log): + log.append(_post("1", "general", "wiseman", "WisemanBot", "hi")) + log.append(_post("2", "general", "cravatt", "CravattBot", "hi")) + log.append(_post("3", "general", None, "Dr PI", "hi", is_bot=False)) + got = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids={"wiseman"}, + ) + assert {p.ts for p in got} == {"1", "3"} + + def test_top_level_posts_unfiltered_when_gate_off(self, log): + log.append(_post("1", "general", "wiseman", "WisemanBot", "hi")) + log.append(_post("2", "general", "cravatt", "CravattBot", "hi")) + got = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", allowed_sender_ids=None + ) + assert {p.ts for p in got} == {"1", "2"} + + def test_tags_filtered(self, log): + log.append(_post("1", "general", "wiseman", "WisemanBot", "hey @SuBot")) + log.append(_post("2", "general", "cravatt", "CravattBot", "hey @SuBot")) + log.append(_post("3", "general", None, "Dr PI", "hey @SuBot", is_bot=False)) + got = log.get_tags_for_agent("SuBot", since=0, allowed_sender_ids={"wiseman"}) + assert {t.ts for t in got} == {"1", "3"} + + def test_replies_filtered(self, log): + log.append(_post("1", "general", "su", "SuBot", "root")) + log.append(_post("2", "general", "wiseman", "WisemanBot", "r", thread_ts="1")) + log.append(_post("3", "general", "cravatt", "CravattBot", "r", thread_ts="1")) + got = log.get_replies_to_agent_posts("su", since=0, allowed_sender_ids={"wiseman"}) + assert {r.ts for r in got} == {"2"} + + def test_has_new_reply_from_other_is_gated(self, log): + log.append(_post("1", "general", "su", "SuBot", "root")) + log.append(_post("2", "general", "cravatt", "CravattBot", "r", thread_ts="1")) + assert log.has_new_reply_from_other("1", "su", 0.0) is True + assert log.has_new_reply_from_other( + "1", "su", 0.0, allowed_sender_ids={"wiseman"} + ) is False + + def test_has_new_reply_ignores_own_messages(self, log): + """Regression: the original returned True for the agent's own reply when the + sender check was ordered after the early return.""" + log.append(_post("1", "general", "su", "SuBot", "root")) + log.append(_post("2", "general", "su", "SuBot", "own follow-up", thread_ts="1")) + assert log.has_new_reply_from_other("1", "su", 0.0) is False + + def test_ungated_methods_take_no_gate_parameter(self): + """Asserted explicitly so widening one becomes a deliberate act.""" + for name in ( + "get_thread_history", "get_thread_message_count", + "get_agent_top_level_posts", "get_last_bot_sender_in_channel", + "get_thread_allowed_agents", "is_funding_thread", "get_entry", + ): + sig = inspect.signature(getattr(MessageLog, name)) + assert "allowed_sender_ids" not in sig.parameters, name + + def test_gated_methods_take_the_gate_parameter(self): + for name in ( + "get_new_top_level_posts", "get_replies_to_agent_posts", + "get_tags_for_agent", "has_new_reply_from_other", + ): + sig = inspect.signature(getattr(MessageLog, name)) + assert "allowed_sender_ids" in sig.parameters, name + + +class TestReadPathInventory: + """Guard: a new public read method must declare its cohort classification. + + Without this the §6 inventory rots the first time someone adds a reader and + forgets the gate — which is exactly how has_new_reply_from_other was missed. + """ + + def test_every_public_read_method_is_classified(self): + unclassified = [] + for name, obj in vars(MessageLog).items(): + if name.startswith("_"): + continue + if not re.match(r"^(get|has|is)_", name) and name != "latest_timestamp": + continue + fn = obj.fget if isinstance(obj, property) else obj + doc = inspect.getdoc(fn) or "" + if "COHORT-GATE: GATED" not in doc and "COHORT-GATE: UNGATED" not in doc: + unclassified.append(name) + assert not unclassified, ( + "MessageLog read methods missing a 'COHORT-GATE: GATED|UNGATED' marker " + f"in their docstring: {unclassified}. See .notes/cohort-system-v2.md §6." + ) + + def test_writes_are_not_gated(self): + for name in ("append", "load_entry", "_record"): + sig = inspect.signature(getattr(MessageLog, name)) + assert "allowed_sender_ids" not in sig.parameters, ( + f"{name} must never take a gate: the log is shared by every agent " + "in the process, so filtering at write filters for all of them" + ) + + +# --------------------------------------------------------------------------- +# §6.1 — stale banked posts +# --------------------------------------------------------------------------- + + +class TestStatePruning: + async def test_interesting_posts_pruned_on_resync(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman", "cravatt"], + membership_rows=[(c1, "su"), (c1, "wiseman")]) + su = eng.agents["su"] + su.state.interesting_posts = [ + PostRef(post_id="1", channel="general", sender_agent_id="wiseman", + content_snippet="mate", posted_at=1.0), + PostRef(post_id="2", channel="general", sender_agent_id="cravatt", + content_snippet="non-mate", posted_at=2.0), + ] + await eng._recompute_allowed_sender_ids() + assert [p.post_id for p in su.state.interesting_posts] == ["1"] + + async def test_pruning_keeps_human_authored_posts(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman"], membership_rows=[(c1, "su"), (c1, "wiseman")]) + su = eng.agents["su"] + su.state.interesting_posts = [ + PostRef(post_id="h", channel="general", sender_agent_id="", + content_snippet="from a PI", posted_at=1.0), + ] + await eng._recompute_allowed_sender_ids() + assert [p.post_id for p in su.state.interesting_posts] == ["h"] + + async def test_no_pruning_when_gate_off(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=False) + eng = _engine(["su"], membership_rows=[]) + eng.agents["su"].state.interesting_posts = [ + PostRef(post_id="1", channel="general", sender_agent_id="anyone", + content_snippet="x", posted_at=1.0), + ] + await eng._recompute_allowed_sender_ids() + assert len(eng.agents["su"].state.interesting_posts) == 1 + + +# --------------------------------------------------------------------------- +# §6.2 — DB-primary read paths +# --------------------------------------------------------------------------- + + +class TestDbPrimaryPaths: + def test_ingestion_is_complete_while_reads_are_filtered(self): + """_poll_inbound_from_db feeds a log shared by every agent. The shared log + must stay complete; only the per-agent read is filtered.""" + log = MessageLog() + log.append(_post("1", "general", "wiseman", "WisemanBot", "a")) + log.append(_post("2", "general", "cravatt", "CravattBot", "b")) + assert len(log) == 2, "ingestion must not drop anything" + gated = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids={"wiseman"}, + ) + ungated = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=None, + ) + assert len(gated) == 1 and len(ungated) == 2 + + def test_null_agent_id_bot_row_does_not_leak(self): + """The shape _poll_inbound_from_db produces from a nullable agent_id.""" + log = MessageLog() + log.append(_post("1", "general", None, "bot", "unattributable", is_bot=True)) + got = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids={"wiseman"}, + ) + assert got == [] + + def test_agent_message_agent_id_is_nullable(self): + """Pins the schema fact the is_bot keying exists for. If this ever becomes + NOT NULL, the fail-closed branch is still correct but no longer load-bearing.""" + from src.models.agent_activity import AgentMessage + assert AgentMessage.__table__.c.agent_id.nullable is True + + def test_log_entry_carries_persisted_visibility(self): + """§7 reads LogEntry.visibility rather than the engine's in-memory channel + map, so it must survive ingestion from another process.""" + from src.models.agent_activity import AgentMessage + assert "visibility" in AgentMessage.__table__.c + assert "visibility" in {f.name for f in __import__("dataclasses").fields(LogEntry)} + + +# --------------------------------------------------------------------------- +# §7 — PI-created private channels outrank the gate +# --------------------------------------------------------------------------- + + +class TestPrivateChannels: + def test_partner_visible_in_pi_created_private_channel(self): + log = MessageLog() + log.append(_post("1", "collab-priv", None, "Dr PI", "work together", + is_bot=False, visibility=VISIBILITY_COLLAB_PRIVATE)) + log.append(_post("2", "collab-priv", "cravatt", "CravattBot", "my angle", + visibility=VISIBILITY_COLLAB_PRIVATE)) + got = log.get_new_top_level_posts( + since=0, channels={"collab-priv"}, exclude_agent_id="su", + allowed_sender_ids=set(), # maximally isolated + ) + assert {p.ts for p in got} == {"1", "2"}, ( + "an explicit PI pairing must not be vetoed by a cohort" + ) + + def test_public_channel_from_same_partner_is_still_filtered(self): + log = MessageLog() + log.append(_post("1", "general", "cravatt", "CravattBot", "public post")) + got = log.get_new_top_level_posts( + since=0, channels={"general"}, exclude_agent_id="su", + allowed_sender_ids=set(), + ) + assert got == [] + + def test_private_channel_tags_pass(self): + log = MessageLog() + log.set_bot_name_map({"subot": "su"}) + log.append(_post("1", "collab-priv", "cravatt", "CravattBot", "hey @SuBot", + visibility=VISIBILITY_COLLAB_PRIVATE)) + got = log.get_tags_for_agent("SuBot", since=0, allowed_sender_ids=set()) + assert len(got) == 1 + + async def test_private_channel_thread_is_never_grandfathered(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + eng._channel_visibility["collab-priv"] = VISIBILITY_COLLAB_PRIVATE + t = _thread(eng.agents["su"], "1", "cravatt", channel="collab-priv") + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is False + + +# --------------------------------------------------------------------------- +# §8 — grandfathering +# --------------------------------------------------------------------------- + + +class TestGrandfathering: + async def test_membership_change_grandfathers_the_thread(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + t = _thread(eng.agents["su"], "1", "cravatt") + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is True + + async def test_resumed_run_grandfathers_rebuilt_threads(self, monkeypatch): + """The rebuild is gate-blind by construction, so the first recompute is + where a restart's inherited partnerships get marked.""" + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman", "cravatt"], + membership_rows=[(c1, "su"), (c1, "wiseman")]) + legal = _thread(eng.agents["su"], "1", "wiseman") + inherited = _thread(eng.agents["su"], "2", "cravatt") + assert eng.agents["su"].allowed_sender_ids is None # pre-recompute: blind + await eng._recompute_allowed_sender_ids() + assert legal.grandfathered is False + assert inherited.grandfathered is True + + async def test_re_permission_clears_the_flag(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + t = _thread(eng.agents["su"], "1", "cravatt") + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is True + eng._fake_db._rows = [(c1, "su"), (c1, "cravatt")] + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is False + + async def test_disabling_isolation_clears_the_flag(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + t = _thread(eng.agents["su"], "1", "cravatt") + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is True + _patch(monkeypatch, cohort_isolation_enabled=False) + await eng._recompute_allowed_sender_ids() + assert t.grandfathered is False + + async def test_grandfathered_thread_loses_reactive_priority(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + _thread(eng.agents["su"], "1", "cravatt", pending=True) + await eng._recompute_allowed_sender_ids() + assert eng._owes_reply(eng.agents["su"]) is False + + async def test_permitted_thread_keeps_reactive_priority(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman"], membership_rows=[(c1, "su"), (c1, "wiseman")]) + _thread(eng.agents["su"], "1", "wiseman", pending=True) + await eng._recompute_allowed_sender_ids() + assert eng._owes_reply(eng.agents["su"]) is True + + async def test_non_cohort_third_party_cannot_manufacture_priority(self, monkeypatch): + """A funding thread is open to all, so a non-cohort agent can post into an + otherwise legal thread. That must not create reactive priority.""" + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman", "cravatt"], + membership_rows=[(c1, "su"), (c1, "wiseman")]) + _thread(eng.agents["su"], "1", "wiseman") + eng.message_log.append(_post("1", "general", "su", "SuBot", ":moneybag: FOA")) + eng.message_log.append( + _post("2", "general", "cravatt", "CravattBot", "me too", thread_ts="1") + ) + eng.agents["su"].state.last_seen_cursor = 0.0 + await eng._recompute_allowed_sender_ids() + assert eng._owes_reply(eng.agents["su"]) is False + + def test_phase4_reads_ungated_so_threads_can_conclude(self): + """Phase 4 must see a grandfathered partner's reply — the thread is open and + entitled to finish. Pinned on the call site, since the whole point of §8 is + that Phase 4 and the scheduler deliberately differ.""" + src = inspect.getsource(SimulationEngine._phase4_reply_threads) + assert "allowed_sender_ids=None" in src + assert "entitled to conclude" in src + + async def test_closed_thread_is_not_grandfathered_or_owed(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + t = _thread(eng.agents["su"], "1", "cravatt", pending=True) + t.status = "closed" + await eng._recompute_allowed_sender_ids() + assert eng._owes_reply(eng.agents["su"]) is False + + +# --------------------------------------------------------------------------- +# §9 — outbound mention hygiene +# --------------------------------------------------------------------------- + + +class TestTagHygiene: + def _eng(self, monkeypatch, allowed): + _patch(monkeypatch, cohort_isolation_enabled=True) + eng = _engine(["su", "wiseman", "cravatt"]) + eng.agents["su"].allowed_sender_ids = allowed + return eng + + def test_no_op_when_gate_off(self, monkeypatch): + eng = self._eng(monkeypatch, None) + text = "Hey @WisemanBot, thoughts?" + assert eng._strip_disallowed_tags(text, eng.agents["su"]) == text + + def test_cohort_mate_mention_survives(self, monkeypatch): + eng = self._eng(monkeypatch, {"su", "wiseman"}) + text = "Hey @WisemanBot, thoughts?" + assert eng._strip_disallowed_tags(text, eng.agents["su"]) == text + + def test_non_mate_mention_is_removed_not_de_atted(self, monkeypatch): + eng = self._eng(monkeypatch, {"su", "wiseman"}) + out = eng._strip_disallowed_tags("Great point @CravattBot, shall we?", + eng.agents["su"]) + assert "CravattBot" not in out + assert "@" not in out + assert out == "Great point, shall we?", out + + def test_self_mention_survives(self, monkeypatch): + eng = self._eng(monkeypatch, set()) + assert eng._strip_disallowed_tags("as @SuBot said", eng.agents["su"]) == ( + "as @SuBot said" + ) + + def test_unknown_bot_name_is_left_alone_and_warned(self, monkeypatch, caplog): + eng = self._eng(monkeypatch, set()) + with caplog.at_level("WARNING"): + out = eng._strip_disallowed_tags("ping @GhostBot", eng.agents["su"]) + assert out == "ping @GhostBot" + assert any("unknown bot name" in r.getMessage().lower() for r in caplog.records) + + def test_strips_are_counted_per_agent(self, monkeypatch): + eng = self._eng(monkeypatch, {"su"}) + eng._strip_disallowed_tags("@WisemanBot @CravattBot hi", eng.agents["su"]) + assert eng._cohort_tags_stripped["su"] == 2 + + def test_no_count_when_nothing_stripped(self, monkeypatch): + eng = self._eng(monkeypatch, {"su", "wiseman"}) + eng._strip_disallowed_tags("@WisemanBot hi", eng.agents["su"]) + assert "su" not in eng._cohort_tags_stripped + + def test_empty_and_none_text(self, monkeypatch): + eng = self._eng(monkeypatch, set()) + assert eng._strip_disallowed_tags(None, eng.agents["su"]) is None + assert eng._strip_disallowed_tags("", eng.agents["su"]) == "" + + def test_all_outbound_paths_are_covered(self): + """The strip lives in _post_message, so every caller inherits it — Phase 4 + replies included, which the original Phase-5-only placement missed.""" + assert "_strip_disallowed_tags" in inspect.getsource( + SimulationEngine._post_message + ) + + def test_indentation_and_code_blocks_survive(self, monkeypatch): + """Regression: an earlier global whitespace normalisation stripped leading + indentation on every line, mangling code blocks and nested bullet lists.""" + eng = self._eng(monkeypatch, {"su", "wiseman"}) + text = ( + "Proposal:\n\n```python\n def f():\n return 1\n```\n\n" + "- point one\n - nested\ncc @CravattBot" + ) + out = eng._strip_disallowed_tags(text, eng.agents["su"]) + assert " def f():" in out + assert " return 1" in out + assert " - nested" in out + assert "CravattBot" not in out + + def test_mention_at_line_start_leaves_no_leading_space(self, monkeypatch): + eng = self._eng(monkeypatch, {"su"}) + assert eng._strip_disallowed_tags("@CravattBot hi", eng.agents["su"]) == "hi" + + def test_trailing_mention_leaves_no_trailing_space(self, monkeypatch): + eng = self._eng(monkeypatch, {"su"}) + assert eng._strip_disallowed_tags("cc @CravattBot", eng.agents["su"]) == "cc" + + def test_email_and_url_are_not_mangled(self, monkeypatch): + """The strip now runs on every outbound message, so a bare '@' inside an + address or URL path must not be read as a mention.""" + eng = self._eng(monkeypatch, {"su"}) + for text in ("mail a@cravattbot.example", "see http://x/@cravattbot"): + assert eng._strip_disallowed_tags(text, eng.agents["su"]) == text + + def test_mention_needs_a_word_boundary(self, monkeypatch): + eng = self._eng(monkeypatch, {"su"}) + assert eng._strip_disallowed_tags("@CravattBotly", eng.agents["su"]) == ( + "@CravattBotly" + ) + + def test_idempotent(self, monkeypatch): + eng = self._eng(monkeypatch, {"su"}) + once = eng._strip_disallowed_tags("hi @CravattBot there", eng.agents["su"]) + twice = eng._strip_disallowed_tags(once, eng.agents["su"]) + assert once == twice + + +# --------------------------------------------------------------------------- +# §10 — scheduler +# --------------------------------------------------------------------------- + + +class TestScheduler: + def test_owed_agent_selected_first(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["su", "wiseman", "cravatt"]) + _thread(eng.agents["wiseman"], "t1", "su", pending=True) + assert eng._select_agent().agent_id == "wiseman" + assert eng._reactive_streak == 1 + + def test_oldest_waiting_owed_agent_wins(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["su", "wiseman"]) + _thread(eng.agents["su"], "t1", "wiseman", pending=True) + _thread(eng.agents["wiseman"], "t2", "su", pending=True) + eng.agents["su"].state.last_selected = 100.0 + eng.agents["wiseman"].state.last_selected = 5.0 + assert eng._select_agent().agent_id == "wiseman" + + def test_excludes_last_llm_caller(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["su", "wiseman"]) + _thread(eng.agents["su"], "t1", "wiseman", pending=True) + _thread(eng.agents["wiseman"], "t2", "su", pending=True) + eng.agents["su"].state.last_selected = 1.0 + eng.agents["wiseman"].state.last_selected = 50.0 + eng._last_llm_caller = "su" + assert eng._select_agent().agent_id == "wiseman" + + def test_no_candidates_returns_none(self, monkeypatch): + _patch(monkeypatch) + assert _engine([])._select_agent() is None + + def test_per_agent_cooldown_is_enforced(self, monkeypatch): + """turn_delay_seconds is a per-agent cooldown at selection time, not a + global sleep.""" + import time + _patch(monkeypatch, turn_delay_seconds=10_000.0) + eng = _engine(["su"]) + eng.agents["su"].state.last_selected = time.time() + assert eng._select_agent() is None + + def test_cooldown_only_sidelines_the_agent_that_just_ran(self, monkeypatch): + import time + _patch(monkeypatch, turn_delay_seconds=10_000.0) + eng = _engine(["su", "wiseman"]) + eng.agents["su"].state.last_selected = time.time() + eng.agents["wiseman"].state.last_selected = 0.0 + picked = eng._select_agent() + assert picked is not None and picked.agent_id == "wiseman" + + def test_cooldown_applies_to_the_reactive_tier_too(self, monkeypatch): + import time + _patch(monkeypatch, turn_delay_seconds=10_000.0) + eng = _engine(["su", "wiseman"]) + _thread(eng.agents["su"], "t1", "wiseman", pending=True) + eng.agents["su"].state.last_selected = time.time() + eng.agents["wiseman"].state.last_selected = 0.0 + assert eng._select_agent().agent_id == "wiseman" + + def test_global_sleep_removed_from_main_loop(self): + # The loop was split out of start() into _run_main_loop() so the + # stall-is-transient contract is reachable from a unit test; check both + # halves so the global sleep cannot reappear in either. + src = inspect.getsource(SimulationEngine.start) + inspect.getsource( + SimulationEngine._run_main_loop + ) + assert "_sleep(settings.turn_delay_seconds)" not in src + assert "enforced at selection time in _turn_eligible" in src + + def test_valve_forces_proactive_at_cap(self, monkeypatch): + _patch(monkeypatch, max_consecutive_reactive_turns=3) + eng = _engine(["su", "wiseman"]) + _thread(eng.agents["wiseman"], "t1", "su", pending=True) + eng._reactive_streak = 3 + assert eng._select_agent() is not None + assert eng._reactive_streak == 0 + + def test_default_valve_is_three(self): + from src.config import Settings + assert Settings.model_fields["max_consecutive_reactive_turns"].default == 3 + + def test_valve_caps_starvation_at_three_to_one(self, monkeypatch): + """At the original default of 8 a live pair took 24 of 27 turns. + + Models the real loop: `start()` advances `last_selected` after every turn, + which is what lets the staleness-weighted proactive tier favour the agents + the reactive pair has been starving. A fake clock is required — with wall + time every delta is sub-second and `max(now - last_selected, 1.0)` clamps + every weight to 1.0, making the proactive tier uniform and the assertion + meaningless. + """ + import random + + import src.agent.simulation as sim + + # The proactive tier is random.choices. Seed it so this bound is a fact + # about the scheduler rather than about today's RNG state. + random.seed(20260730) + _patch(monkeypatch, max_consecutive_reactive_turns=3) + clock = [1000.0] + monkeypatch.setattr(sim.time, "time", lambda: clock[0]) + + eng = _engine(["su", "wiseman", "a", "b", "c"]) + _thread(eng.agents["su"], "t1", "wiseman", pending=True) + _thread(eng.agents["wiseman"], "t2", "su", pending=True) + picks = [] + for _ in range(40): + got = eng._select_agent() + picks.append(got.agent_id) + eng._last_llm_caller = got.agent_id + got.state.last_selected = clock[0] # as start() does + clock[0] += 10.0 + pair = sum(1 for p in picks if p in {"su", "wiseman"}) + assert pair <= 32, f"{pair}/40 went to the live pair: {picks}" + assert pair >= 24, "the reactive tier should still dominate" + starved = {a for a in ("a", "b", "c") if a in picks} + assert starved == {"a", "b", "c"}, ( + f"every idle agent must get a turn within 40 selections, got {starved}" + ) + + def test_selection_counters_advance(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["su", "wiseman"]) + _thread(eng.agents["wiseman"], "t1", "su", pending=True) + eng._select_agent() + assert eng._reactive_selections == 1 and eng._proactive_selections == 0 + eng.agents["wiseman"].state.active_threads.clear() + eng._select_agent() + assert eng._proactive_selections == 1 + + def test_budget_still_filters(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["su"], budget_cap=1) + eng.agents["su"].api_call_count = 5 + assert eng._select_agent() is None + + +# --------------------------------------------------------------------------- +# §13.1 — provenance +# --------------------------------------------------------------------------- + + +class TestTopologySnapshot: + async def test_snapshot_records_the_applied_gate(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + c1 = uuid.uuid4() + eng = _engine(["su", "wiseman", "cravatt"], + membership_rows=[(c1, "su"), (c1, "wiseman")]) + await eng._recompute_allowed_sender_ids() + snap = eng.cohort_topology_snapshot() + assert snap["cohort_isolation_enabled"] is True + assert snap["cohort_default_policy"] == POLICY_ISOLATED + assert snap["agents"]["su"] == ["su", "wiseman"] + assert snap["agents"]["cravatt"] == [] + assert snap["preflight_error"] is None + + async def test_snapshot_records_a_preflight_override(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True, + cohort_default_policy=POLICY_ISOLATED) + eng = _engine(["su"], membership_rows=[], cohort_count=0) + await eng._recompute_allowed_sender_ids() + snap = eng.cohort_topology_snapshot() + assert snap["preflight_error"] is not None + assert snap["agents"]["su"] is None + + async def test_snapshot_carries_counters(self, monkeypatch): + _patch(monkeypatch, cohort_isolation_enabled=True) + c1 = uuid.uuid4() + eng = _engine(["su", "cravatt"], membership_rows=[(c1, "su")]) + await eng._recompute_allowed_sender_ids() + eng._cohort_tags_stripped["su"] = 4 + _thread(eng.agents["su"], "1", "cravatt", grandfathered=True) + c = eng.cohort_topology_snapshot()["counters"] + assert c["tags_stripped"] == {"su": 4} + assert c["grandfathered_threads"] == ["su:1"] + + async def test_snapshot_is_json_serialisable(self, monkeypatch): + import json + _patch(monkeypatch, cohort_isolation_enabled=True) + c1 = uuid.uuid4() + eng = _engine(["su"], membership_rows=[(c1, "su")]) + await eng._recompute_allowed_sender_ids() + json.dumps(eng.cohort_topology_snapshot()) + + +# --------------------------------------------------------------------------- +# §14 — migration hygiene +# --------------------------------------------------------------------------- + + +class TestMigrationHygiene: + VERSIONS = pathlib.Path(__file__).resolve().parents[2] / "alembic" / "versions" + + def _revisions(self): + out = {} + for f in sorted(self.VERSIONS.glob("*.py")): + m = re.search(r'^revision:?\s*(?::\s*str\s*)?=\s*["\'](.+?)["\']', + f.read_text(), re.M) + if m: + out.setdefault(m.group(1), []).append(f.name) + return out + + def test_no_duplicate_revision_ids(self): + dupes = {r: fs for r, fs in self._revisions().items() if len(fs) > 1} + assert not dupes, ( + f"duplicate alembic revision ids {dupes} — Alembic keeps only the " + "last-sorted file, silently skipping the other while stamping the DB " + "as fully migrated. See .notes/cohort-system-v2.md §14." + ) + + def test_exactly_one_head(self): + revs, downs = {}, set() + for f in sorted(self.VERSIONS.glob("*.py")): + src = f.read_text() + r = re.search(r'^revision:?\s*(?::\s*str\s*)?=\s*["\'](.+?)["\']', src, re.M) + d = re.search(r'^down_revision[^=]*=\s*["\'](.+?)["\']', src, re.M) + if r: + revs[r.group(1)] = f.name + if d: + downs.add(d.group(1)) + heads = sorted(set(revs) - downs) + assert len(heads) == 1, f"expected 1 alembic head, found {heads}" + + def test_cohort_migration_is_on_the_current_head(self): + f = self.VERSIONS / "0022_add_cohorts.py" + assert f.exists(), "cohort migration must be renumbered to 0022" + src = f.read_text() + assert 'revision: str = "0022"' in src + assert 'down_revision: Union[str, None] = "0021"' in src + + def test_cohort_downgrade_is_idempotent(self): + """A rollback must not wedge on an object a partial upgrade never created.""" + src = (self.VERSIONS / "0022_add_cohorts.py").read_text() + downgrade = src[src.index("def downgrade"):] + drops = re.findall(r"op\.drop_(?:table|index)\(", downgrade) + assert downgrade.count("if_exists=True") == len(drops), ( + "every drop in the cohort downgrade needs if_exists=True" + ) + + def test_head_is_the_expected_revision(self): + """Positive control for the single-head and no-duplicates assertions. + + Both of those are absences. An empty versions directory, or a parser whose + regex stopped matching the file format, would satisfy "exactly one head" and + "no duplicates" trivially. This pins what the parser actually found. + """ + revs = self._revisions() + assert len(revs) >= 22, ( + f"only {len(revs)} revisions parsed from {self.VERSIONS} — the regex has " + "probably stopped matching, which would make every other assertion in " + "this class vacuous" + ) + assert "0022" in revs + assert revs["0022"] == ["0022_add_cohorts.py"] + + def test_ci_script_gates_on_alembic_before_running_tests(self): + """The durable fix for the collision is the gate, not the one renumbered file. + + A future branch that assigns its revision id at branch time rather than at + merge time reintroduces the whole failure — clean git merge, green pytest, + broken deploy. Only the gate catches that, so the gate itself is pinned here. + """ + ci = (pathlib.Path(__file__).resolve().parents[2] / "scripts" / "ci.sh").read_text() + assert "alembic heads" in ci, "the single-head check is missing from scripts/ci.sh" + assert "uniq -d" in ci, "the duplicate-revision-id check is missing" + # Order matters: a broken migration chain must be reported in seconds, not + # after the full suite (which passes regardless — that is the whole problem). + assert ci.index("uniq -d") < ci.index("-m pytest") + assert ci.index("alembic heads") < ci.index("-m pytest") diff --git a/tests/unit/test_config_secret_redaction.py b/tests/unit/test_config_secret_redaction.py index 45a94bf..11874aa 100644 --- a/tests/unit/test_config_secret_redaction.py +++ b/tests/unit/test_config_secret_redaction.py @@ -1,6 +1,14 @@ """Settings repr()/str() must not leak credentials (SEC-19).""" -from src.config import Settings +import pytest + +from src.config import _SECRET_NAME_HINTS, Settings, _redact_url_credentials + +# A DSN with the password embedded in the userinfo — the shape the app ships with +# (docker-compose sets DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi). +# `database_url` matches none of the credential name hints, so before the positional +# URL redaction the whole DSN, password included, appeared verbatim in repr(settings). +LEAKY_DSN = "postgresql+asyncpg://copi:sup3rs3cr3t@postgres:5432/copi" def _settings(): @@ -50,3 +58,190 @@ def test_empty_secret_not_labeled_redacted(): args = dict(s.__repr_args__()) assert args["secret_key"] == "" # empty -> not masked assert args["slack_bot_token_su"] == "***REDACTED***" # non-empty -> masked + + +# --- credentials embedded in a URL/DSN -------------------------------------------- + + +def test_database_url_password_is_redacted(): + """The leak this file missed: DATABASE_URL carries the DB password in its + userinfo, and `database_url` matches no credential name hint.""" + s = Settings(_env_file=None, database_url=LEAKY_DSN) + for rendered in (repr(s), str(s)): + assert "sup3rs3cr3t" not in rendered + assert LEAKY_DSN not in rendered + + +def test_only_the_password_component_of_a_dsn_is_masked(): + """Positional, not whole-value, masking. An operator debugging a deploy needs to + see which host/port/database the app is pointed at; only the password is secret. + (Same choice as SQLAlchemy's URL.render_as_string(hide_password=True).)""" + r = repr(Settings(_env_file=None, database_url=LEAKY_DSN)) + assert "postgresql+asyncpg://copi:***REDACTED***@postgres:5432/copi" in r + + +def test_dsn_with_no_userinfo_is_shown_in_full(): + """Documented decision: a URL with no userinfo holds no credential, so it is NOT + masked. Masking it would destroy the one field an operator most needs in a deploy + postmortem, and would make the mask ambiguous about whether a password exists.""" + plain = "postgresql://postgres:5432/copi" + r = repr(Settings(_env_file=None, database_url=plain)) + assert plain in r + + +def test_dsn_with_a_bare_username_keeps_the_username_visible(): + """A userinfo with no ":" is a username, not a credential. A URL whose userinfo + *is* the credential would live in a *_token field and be masked whole by name.""" + r = repr(Settings(_env_file=None, database_url="postgresql://copi@postgres/copi")) + assert "postgresql://copi@postgres/copi" in r + + +def test_dsn_with_an_empty_password_is_not_labeled_redacted(): + """Mirrors test_empty_secret_not_labeled_redacted for the positional path: the + mask must mean "a real value is hidden here".""" + r = repr(Settings(_env_file=None, database_url="postgresql://copi:@postgres/copi")) + assert "postgresql://copi:@postgres/copi" in r + + +def test_a_password_in_the_dsn_query_string_is_redacted(): + """libpq/asyncpg also accept `?password=`. Control in the same assertion: a + non-credential parameter next to it stays visible.""" + dsn = "postgresql://postgres:5432/copi?sslmode=require&password=hunter2" + r = repr(Settings(_env_file=None, database_url=dsn)) + assert "hunter2" not in r + assert "sslmode=require" in r + assert "password=***REDACTED***" in r + + +def test_a_key_file_path_in_the_dsn_query_string_stays_visible(): + """Over-redaction is a real cost: `?sslkey=` is a filename an operator needs, not + a secret, so the query-parameter hints are narrower than the field-name hints.""" + dsn = "postgresql://postgres:5432/copi?sslkey=/etc/ssl/client.key" + r = repr(Settings(_env_file=None, database_url=dsn)) + assert "/etc/ssl/client.key" in r + + +def test_reading_database_url_still_returns_the_real_dsn(): + """Redaction is display-only — create_async_engine(settings.database_url) must + still get a connectable DSN. No field changed type.""" + s = Settings(_env_file=None, database_url=LEAKY_DSN) + assert s.database_url == LEAKY_DSN + assert "REDACTED" not in s.database_url + + +# --- systematic sweep over every string field ------------------------------------- + +# Every `str`-annotated field whose value cannot carry a credential, and is therefore +# expected to render in the clear. Criterion: the value is a public identifier, an +# infrastructure name, or an operator-facing diagnostic — never a bearer credential, +# a signing key, or a password. Adding a field to Settings that renders in the clear +# forces an edit here, i.e. an explicit classification. +NON_SECRET_STR_FIELDS = { + # Comma-separated daily-audit recipient emails (copi-prod's 9ab5555) — + # addresses, not credentials. + "audit_recipients", + "environment", + "orcid_client_id", # OAuth *public* client id; ships in the browser redirect + "orcid_redirect_uri", + "database_url", # a plain sentinel has no userinfo -> nothing to mask + "ncbi_contact_email", + "base_url", + "aws_region", + "ses_sender_email", + "ses_reply_domain", + "ses_inbound_s3_bucket", + "ses_inbound_s3_prefix", + "outbound_email_allowlist", + "llm_profile_model", + "llm_agent_model", + "llm_agent_model_opus", + "llm_agent_model_sonnet", +} + + +def _str_field_names(): + return [n for n, f in Settings.model_fields.items() if f.annotation is str] + + +def _sweep_settings(value_for): + return Settings(_env_file=None, **{n: value_for(n) for n in _str_field_names()}) + + +def test_every_string_field_is_classified_secret_or_not(): + """Fills every str field with a unique sentinel and asserts that exactly the + documented non-secret fields survive in the repr. Fails both ways: a new + credential field whose name misses the hints shows up as an unexpected leak, and + a repr that masked everything shows up as a missing control.""" + s = _sweep_settings(lambda n: f"sentinelvalue-{n}") + rendered = repr(s) + str(s) + visible = {n for n in _str_field_names() if f"sentinelvalue-{n}" in rendered} + assert visible == NON_SECRET_STR_FIELDS + + +def test_no_string_field_leaks_a_password_embedded_in_a_url(): + """Second sweep: every str field gets a DSN carrying a password in its userinfo. + Catches such a field present or future, regardless of its name. + + Scope, stated precisely because the obvious reading is wider than the truth: this + sweeps the two places `_redact_url_credentials` looks — the userinfo and the query + string. A URL whose credential is a PATH SEGMENT is not covered; see + test_a_credential_in_a_url_path_is_a_known_gap below.""" + s = _sweep_settings(lambda n: f"postgresql://user:pw-{n}@host:5432/db") + rendered = repr(s) + str(s) + leaked = [n for n in _str_field_names() if f"pw-{n}" in rendered] + assert leaked == [] + # Control: the sweep really did populate the object. + assert s.database_url == "postgresql://user:pw-database_url@host:5432/db" + + +def test_a_credential_in_a_url_path_is_a_known_gap(): + """Pins the one credential shape the positional path does NOT mask, so it is a + recorded limitation rather than a surprise. + + A Slack/Discord incoming-webhook URL, an S3 presigned URL and a Twilio-style + callback all carry their secret in the PATH, not the userinfo or the query string. + `_redact_url_credentials` masks neither, deliberately: `base_url` and + `orcid_redirect_uri` have paths that an operator needs to read, and there is no + way to tell a secret path segment from a route without knowing the field. + + No Settings field has this shape today — `test_every_string_field_is_classified_ + secret_or_not` is what keeps that true, because a new field renders in the clear + only if someone adds it to NON_SECRET_STR_FIELDS in the same diff. If one is ever + added whose name misses `_SECRET_NAME_HINTS` (e.g. `slack_incoming_webhook`), the + right fix is the name hint, not a path heuristic.""" + webhook = "https://hooks.slack.com/services/T00000/B00000/xxxxSECRETxxxx" + assert _redact_url_credentials(webhook) == webhook + + # And why that is not a live leak: every str field is either masked whole by name + # or explicitly classified non-secret. There is no third, unreviewed category for a + # path-credential field to hide in. + unclassified = [ + n for n in _str_field_names() + if n not in NON_SECRET_STR_FIELDS + and not any(h in n.lower() for h in _SECRET_NAME_HINTS) + ] + assert unclassified == [], ( + "str field(s) that are neither name-masked nor listed as non-secret — a " + f"path-credential field could hide here: {unclassified}" + ) + + +def test_sweep_covers_the_bot_tokens_and_the_dsn(): + """Control for the sweep helpers themselves — a _str_field_names() that returned + [] would make both sweeps vacuous.""" + names = _str_field_names() + assert "database_url" in names + assert "secret_key" in names + assert len([n for n in names if n.startswith("slack_bot_token_")]) > 100 + + +@pytest.mark.parametrize("render", [repr, str], ids=["repr", "str"]) +def test_both_repr_and_str_route_through_repr_args(render): + """pydantic v2 implements BaseModel.__str__ via __repr_str__ -> __repr_args__, so + one override covers both. Asserted rather than assumed.""" + s = Settings(_env_file=None, secret_key="supersecretvalue", database_url=LEAKY_DSN) + out = render(s) + assert "supersecretvalue" not in out + assert "sup3rs3cr3t" not in out + assert "***REDACTED***" in out + assert "http" in out or "postgres" in out # control: something is still rendered diff --git a/tests/unit/test_email_inbound_hardening.py b/tests/unit/test_email_inbound_hardening.py new file mode 100644 index 0000000..273aef4 --- /dev/null +++ b/tests/unit/test_email_inbound_hardening.py @@ -0,0 +1,255 @@ +"""Hardening for inbound email reply processing. + +These pin the defects found while investigating the dead prod reply flow +(2026-08-11): a sender-forged ``Authentication-Results: ... pass`` header +defeated the SEC-5 anti-spoofing gate, HTML-only replies were silently +dropped, auto-responders could loop with the help email, the declared +per-token rate limit was never enforced, and a poison message in the inbound +bucket was retried forever. +""" + +import email + +import pytest + +import src.services.email_inbound as inbound +from src.services.email_inbound import ( + MAX_REPLIES_PER_TOKEN_PER_HOUR, + _authentication_results_ok, + _extract_reply_body, + _reply_rate_ok, + poll_inbound_emails, + process_inbound_email, +) + + +def _msg(raw: str) -> email.message.Message: + return email.message_from_string(raw) + + +# --- Authentication-Results: only SES's own (topmost) header is trusted ----- + + +def test_forged_pass_header_below_ses_fail_is_rejected(): + """SES prepends its header on receipt, so a sender-supplied pass sits below + it. Merging verdicts across headers let the forged pass win (SEC-5).""" + raw = ( + "Authentication-Results: amazonses.com; spf=fail smtp.mailfrom=evil.com; " + "dkim=none; dmarc=fail header.from=scripps.edu\n" + "Authentication-Results: amazonses.com; spf=pass; dkim=pass; dmarc=pass\n" + "From: pi@scripps.edu\n\nbody" + ) + assert _authentication_results_ok(_msg(raw)) is False + + +def test_verdicts_below_the_topmost_header_are_ignored_entirely(): + raw = ( + "Authentication-Results: amazonses.com; spf=pass smtp.mailfrom=scripps.edu; " + "dkim=pass; dmarc=pass header.from=scripps.edu\n" + "Authentication-Results: evil.example; spf=fail; dkim=fail; dmarc=fail\n" + "From: pi@scripps.edu\n\nbody" + ) + assert _authentication_results_ok(_msg(raw)) is True + + +def test_topmost_header_with_foreign_authserv_id_is_rejected(): + """Everything on our receipt path is stamped by amazonses.com; anything + else means the message did not transit SES receiving.""" + raw = ( + "Authentication-Results: mx.evil.example; spf=pass; dkim=pass; dmarc=pass\n" + "From: pi@scripps.edu\n\nbody" + ) + assert _authentication_results_ok(_msg(raw)) is False + + +# --- HTML-only replies are not silently dropped ------------------------------ + + +def test_html_only_reply_body_falls_back_to_stripped_html(): + raw = ( + "From: pi@scripps.edu\n" + "MIME-Version: 1.0\n" + 'Content-Type: multipart/alternative; boundary="xyz"\n' + "\n" + "--xyz\n" + 'Content-Type: text/html; charset="UTF-8"\n' + "\n" + "<div dir=\"ltr\">4 — excellent, go ahead!<br></div>\n" + '<div class="gmail_quote"><blockquote>quoted proposal text ' + "1 = Not a good idea</blockquote></div>\n" + "\n" + "--xyz--\n" + ) + body = _extract_reply_body(_msg(raw)) + assert "4" in body and "excellent" in body + assert "Not a good idea" not in body # quoted HTML must not leak through + + +def test_singlepart_html_reply_body_is_extracted(): + raw = ( + "From: pi@scripps.edu\n" + 'Content-Type: text/html; charset="UTF-8"\n' + "\n" + "<p>2 & please focus on assay development</p>\n" + ) + body = _extract_reply_body(_msg(raw)) + assert "2 & please focus on assay development" in body + + +def test_plain_text_part_still_wins_over_html(): + raw = ( + "From: pi@scripps.edu\n" + "MIME-Version: 1.0\n" + 'Content-Type: multipart/alternative; boundary="qq"\n' + "\n" + "--qq\n" + 'Content-Type: text/plain; charset="UTF-8"\n' + "\n" + "3 sounds great\n" + "\n" + "--qq\n" + 'Content-Type: text/html; charset="UTF-8"\n' + "\n" + "<div>3 sounds great</div>\n" + "\n" + "--qq--\n" + ) + assert _extract_reply_body(_msg(raw)) == "3 sounds great" + + +# --- Auto-submitted mail is dropped before any processing -------------------- + + +_SES_PASS = "Authentication-Results: amazonses.com; spf=pass; dkim=pass; dmarc=pass\n" + + +async def test_auto_submitted_reply_is_ignored_before_touching_the_db(): + """RFC 3834: an OOO auto-reply answering our help email must not trigger + another help email (mail loop). db=None proves the early return.""" + raw = ( + _SES_PASS + + "Auto-Submitted: auto-replied\n" + "From: pi@scripps.edu\n" + "To: review+sometoken@reply.copi.science\n" + "\n" + "I am out of the office.\n" + ).encode() + await process_inbound_email(raw, db=None) # must not raise + + +async def test_auto_submitted_no_is_not_treated_as_an_auto_reply(): + """``Auto-Submitted: no`` explicitly marks human-generated mail; it must + proceed into normal processing (here: to the token lookup, which needs a + db — the AttributeError on db=None is the evidence it got past the gate).""" + raw = ( + _SES_PASS + + "Auto-Submitted: no\n" + "From: pi@scripps.edu\n" + "To: review+sometoken@reply.copi.science\n" + "\n" + "3 great idea\n" + ).encode() + with pytest.raises(AttributeError): + await process_inbound_email(raw, db=None) + + +# --- The declared per-token rate limit is enforced --------------------------- + + +def test_reply_rate_limit_blocks_the_11th_reply_in_an_hour(monkeypatch): + monkeypatch.setattr(inbound, "_RECENT_REPLY_TIMES", {}) + token = "tok-" + "x" * 60 + base = 1_000_000.0 + for i in range(MAX_REPLIES_PER_TOKEN_PER_HOUR): + assert _reply_rate_ok(token, now=base + i) is True + assert _reply_rate_ok(token, now=base + 60) is False + + +def test_reply_rate_limit_window_slides(monkeypatch): + monkeypatch.setattr(inbound, "_RECENT_REPLY_TIMES", {}) + token = "tok-" + "y" * 60 + base = 2_000_000.0 + for i in range(MAX_REPLIES_PER_TOKEN_PER_HOUR): + assert _reply_rate_ok(token, now=base + i) is True + # An hour later the old entries have aged out. + assert _reply_rate_ok(token, now=base + 3601) is True + + +# --- Poison messages are quarantined, not retried forever -------------------- + + +class _FakeS3: + """Just enough of the S3 client for poll_inbound_emails.""" + + def __init__(self, keys): + self.objects = {k: b"raw email bytes" for k in keys} + self.copied: list[tuple[str, str]] = [] + self.deleted: list[str] = [] + + def list_objects_v2(self, Bucket, Prefix, MaxKeys): + return { + "Contents": [{"Key": k} for k in sorted(self.objects)], + "KeyCount": len(self.objects), + } + + def get_object(self, Bucket, Key): + import io + + return {"Body": io.BytesIO(self.objects[Key])} + + def copy_object(self, Bucket, CopySource, Key): + self.copied.append((CopySource["Key"], Key)) + self.objects[Key] = self.objects[CopySource["Key"]] + + def delete_object(self, Bucket, Key): + self.deleted.append(Key) + self.objects.pop(Key, None) + + +class _NullSessionFactory: + def __call__(self): + return self + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def commit(self): + pass + + +async def test_poison_email_is_quarantined_after_repeated_failures(monkeypatch): + fake = _FakeS3(["inbound/poison"]) + monkeypatch.setattr("boto3.client", lambda *a, **k: fake) + monkeypatch.setattr(inbound, "_S3_FAILURE_COUNTS", {}) + + async def _boom(raw, db): + raise RuntimeError("unparseable in a way that always raises") + + monkeypatch.setattr(inbound, "process_inbound_email", _boom) + + for _ in range(inbound.MAX_S3_PROCESS_ATTEMPTS): + assert await poll_inbound_emails(_NullSessionFactory()) == 0 + + assert fake.copied == [("inbound/poison", "failed/poison")] + assert fake.deleted == ["inbound/poison"] + # Quarantined: the next poll sees only failed/ (outside the prefix filter + # in real S3; the fake returns everything, so assert the key is gone). + assert "inbound/poison" not in fake.objects + + +async def test_a_transient_failure_is_retried_not_quarantined(monkeypatch): + fake = _FakeS3(["inbound/flaky"]) + monkeypatch.setattr("boto3.client", lambda *a, **k: fake) + monkeypatch.setattr(inbound, "_S3_FAILURE_COUNTS", {}) + + async def _boom(raw, db): + raise RuntimeError("db briefly down") + + monkeypatch.setattr(inbound, "process_inbound_email", _boom) + await poll_inbound_emails(_NullSessionFactory()) + + assert fake.copied == [] + assert "inbound/flaky" in fake.objects # still there for the next poll diff --git a/tests/unit/test_email_reply_solicitation.py b/tests/unit/test_email_reply_solicitation.py new file mode 100644 index 0000000..9b3022c --- /dev/null +++ b/tests/unit/test_email_reply_solicitation.py @@ -0,0 +1,161 @@ +"""Reply-soliciting email copy must be gated on inbound email being enabled. + +Prod (2026-08-11): review emails told PIs "reply to this email to rate it" +while ENABLE_INBOUND_EMAIL was off and the reply pipeline (MX record, S3 +bucket, receipt rule) did not exist — every PI who replied got silence plus a +bounce. Until inbound is provisioned AND enabled, outbound mail must direct +PIs to the web dashboard only, and must not carry a Reply-To pointing at the +dead reply domain. +""" + +import email +import uuid +from types import SimpleNamespace + +import pytest + +from src.config import get_settings +from src.services.email import build_welcome_email +from src.services.email_notifications import ( + _send_new_proposal_email, + send_proposal_notification, +) + + +class _SESRecorder: + def __init__(self): + self.raw_messages: list[str] = [] + + def send_raw_email(self, **kwargs): + self.raw_messages.append(kwargs["RawMessage"]["Data"]) + return {"MessageId": "m-1"} + + +class _FakeDb: + def __init__(self): + self.added = [] + + def add(self, obj): + self.added.append(obj) + + async def flush(self): + pass + + +@pytest.fixture +def ses(monkeypatch): + recorder = _SESRecorder() + monkeypatch.setattr("boto3.client", lambda *a, **k: recorder) + monkeypatch.setattr(get_settings(), "outbound_email_allowlist", "") + return recorder + + +def _lab(): + user = SimpleNamespace(id=uuid.uuid4(), email="pi@lab.test", name="Ada Alpha") + agent = SimpleNamespace(id=uuid.uuid4(), agent_id="alpha", bot_name="AlphaBot") + td = SimpleNamespace( + id=uuid.uuid4(), summary_text="A joint proposal.", channel="degrader-chem" + ) + return user, agent, td + + +def _parts(raw: str) -> tuple[email.message.Message, str, str]: + msg = email.message_from_string(raw) + text = html = "" + for part in msg.walk(): + if part.get_content_type() == "text/plain": + text = part.get_payload(decode=True).decode("utf-8") + elif part.get_content_type() == "text/html": + html = part.get_payload(decode=True).decode("utf-8") + return msg, text, html + + +# --- proposal_review reminder ------------------------------------------------ + + +async def test_review_reminder_is_web_only_while_inbound_is_disabled( + ses, monkeypatch +): + monkeypatch.setattr(get_settings(), "enable_inbound_email", False) + user, agent, td = _lab() + + ok = await send_proposal_notification( + user=user, thread_decision=td, agent=agent, + other_bot_name="BetaBot", total_unreviewed=1, db=_FakeDb(), + ) + + assert ok is True + msg, text, html = _parts(ses.raw_messages[0]) + assert msg["Reply-To"] is None + assert "Reply to this email" not in text + assert "Reply to this email" not in html + assert "/agent/alpha/dashboard" in text # the web path remains + + +async def test_review_reminder_solicits_replies_when_inbound_is_enabled( + ses, monkeypatch +): + monkeypatch.setattr(get_settings(), "enable_inbound_email", True) + user, agent, td = _lab() + + await send_proposal_notification( + user=user, thread_decision=td, agent=agent, + other_bot_name="BetaBot", total_unreviewed=1, db=_FakeDb(), + ) + + msg, text, html = _parts(ses.raw_messages[0]) + assert msg["Reply-To"].startswith("review+") + assert msg["Reply-To"].endswith(f"@{get_settings().ses_reply_domain}") + assert "Reply to this email" in text + + +# --- new_proposal alert -------------------------------------------------------- + + +async def test_new_proposal_alert_is_web_only_while_inbound_is_disabled( + ses, monkeypatch +): + monkeypatch.setattr(get_settings(), "enable_inbound_email", False) + user, agent, td = _lab() + + ok = await _send_new_proposal_email(user, td, agent, "BetaBot", _FakeDb()) + + assert ok is True + msg, text, html = _parts(ses.raw_messages[0]) + assert msg["Reply-To"] is None + assert "Reply to this email" not in text + assert "Reply to this email" not in html + assert "/agent/alpha/dashboard" in text + + +async def test_new_proposal_alert_solicits_replies_when_inbound_is_enabled( + ses, monkeypatch +): + monkeypatch.setattr(get_settings(), "enable_inbound_email", True) + user, agent, td = _lab() + + await _send_new_proposal_email(user, td, agent, "BetaBot", _FakeDb()) + + msg, text, _ = _parts(ses.raw_messages[0]) + assert msg["Reply-To"].startswith("review+") + assert "Reply to this email" in text + + +# --- welcome email ------------------------------------------------------------- + + +def test_welcome_email_omits_reply_instructions_while_inbound_is_disabled( + monkeypatch, +): + monkeypatch.setattr(get_settings(), "enable_inbound_email", False) + _, msg = build_welcome_email("pi@lab.test", "Ada") + _, text, html = _parts(msg.as_string()) + assert "Reply with a rating" not in text + assert "review" in text.lower() # web review guidance remains + + +def test_welcome_email_describes_replying_when_inbound_is_enabled(monkeypatch): + monkeypatch.setattr(get_settings(), "enable_inbound_email", True) + _, msg = build_welcome_email("pi@lab.test", "Ada") + _, text, _ = _parts(msg.as_string()) + assert "Reply with a rating" in text diff --git a/tests/unit/test_email_templates.py b/tests/unit/test_email_templates.py index 15aef98..b2ae177 100644 --- a/tests/unit/test_email_templates.py +++ b/tests/unit/test_email_templates.py @@ -70,51 +70,72 @@ def test_welcome_email_uses_shared_footer(): def test_delegate_invitation_uses_shared_branding(monkeypatch): """Transactional invite shares the wrapper + tagline but has no unsubscribe.""" - captured = {} + from src.config import get_settings - class _FakeSES: - def send_email(self, **kwargs): - captured["html"] = kwargs["Message"]["Body"]["Html"]["Data"] + # Hermetic: the test's premise is an unrestricted recipient allowlist (the + # field's own default, "" = send to everyone). Pin it rather than inherit + # whatever OUTBOUND_EMAIL_ALLOWLIST the deployed .env on this host sets — + # otherwise send_delegate_invitation silently no-ops and returns False. + monkeypatch.setenv("OUTBOUND_EMAIL_ALLOWLIST", "") + get_settings.cache_clear() + try: + captured = {} - import boto3 + class _FakeSES: + def send_email(self, **kwargs): + captured["html"] = kwargs["Message"]["Body"]["Html"]["Data"] - monkeypatch.setattr(boto3, "client", lambda *a, **k: _FakeSES()) + import boto3 - assert send_delegate_invitation( - "colleague@example.com", "Dr. PI", "PIBot", "https://copi.science/invite/abc" - ) - html = captured["html"] - assert html.lstrip().startswith('<div style="font-family') - assert FOOTER_TAGLINE in html - assert "Unsubscribe" not in html + monkeypatch.setattr(boto3, "client", lambda *a, **k: _FakeSES()) + + assert send_delegate_invitation( + "colleague@example.com", "Dr. PI", "PIBot", "https://copi.science/invite/abc" + ) + html = captured["html"] + assert html.lstrip().startswith('<div style="font-family') + assert FOOTER_TAGLINE in html + assert "Unsubscribe" not in html + finally: + get_settings.cache_clear() def test_delegate_invitation_escapes_untrusted_names(monkeypatch): """PI-chosen pi_name/bot_name must be HTML-escaped in the invite body (SEC-13).""" - captured = {} - - class _FakeSES: - def send_email(self, **kwargs): - captured["html"] = kwargs["Message"]["Body"]["Html"]["Data"] - captured["subject"] = kwargs["Message"]["Subject"]["Data"] - - import boto3 - - monkeypatch.setattr(boto3, "client", lambda *a, **k: _FakeSES()) - - assert send_delegate_invitation( - "colleague@example.com", - '<img src=x onerror=alert(1)>', - '<script>alert(2)</script>', - "https://copi.science/invite/abc", - ) - html = captured["html"] - assert "<img src=x onerror=alert(1)>" not in html - assert "<script>alert(2)</script>" not in html - assert "<img src=x onerror=alert(1)>" in html - assert "<script>alert(2)</script>" in html - # Subject is plain text (not HTML), but must not carry injected newlines. - assert "\n" not in captured["subject"] and "\r" not in captured["subject"] + from src.config import get_settings + + # Hermetic for the same reason as the branding test above: pin the allowlist + # to its default so this test's arbitrary example.com recipient is accepted + # regardless of the deployed .env's OUTBOUND_EMAIL_ALLOWLIST. + monkeypatch.setenv("OUTBOUND_EMAIL_ALLOWLIST", "") + get_settings.cache_clear() + try: + captured = {} + + class _FakeSES: + def send_email(self, **kwargs): + captured["html"] = kwargs["Message"]["Body"]["Html"]["Data"] + captured["subject"] = kwargs["Message"]["Subject"]["Data"] + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda *a, **k: _FakeSES()) + + assert send_delegate_invitation( + "colleague@example.com", + '<img src=x onerror=alert(1)>', + '<script>alert(2)</script>', + "https://copi.science/invite/abc", + ) + html = captured["html"] + assert "<img src=x onerror=alert(1)>" not in html + assert "<script>alert(2)</script>" not in html + assert "<img src=x onerror=alert(1)>" in html + assert "<script>alert(2)</script>" in html + # Subject is plain text (not HTML), but must not carry injected newlines. + assert "\n" not in captured["subject"] and "\r" not in captured["subject"] + finally: + get_settings.cache_clear() @pytest.mark.asyncio diff --git a/tests/unit/test_hub_budget_scheduler.py b/tests/unit/test_hub_budget_scheduler.py new file mode 100644 index 0000000..207cd35 --- /dev/null +++ b/tests/unit/test_hub_budget_scheduler.py @@ -0,0 +1,730 @@ +"""Load-proportional budget and scheduling for star topologies. + +Implements the test plan in docs/specs/2026-08-06-hub-budget-scheduler-design.md +§8. Organised by design section so a failure names the rule it broke: + +- TestAgentLoad §4.1 the shared load signal +- TestRoleRateOverride §4.4 optional per-role allowance +- TestCallLedger §4.2 record_api_call maintains both counters +- TestRateLimiter §4.2 sliding-window eligibility, and that it self-heals +- TestRestartRebuild §4.2 step 4b repopulates call_times from llm_call_logs +- TestScheduler §4.3 load-proportional weight, reactive tiebreak +- TestStallIsTransient F1 a throttled roster must NOT end the run +- TestPIHandlerAccounting F2 PI-DM LLM calls go through record_api_call +- TestRateSettingGuards F4 non-positive rate settings are clamped, loudly +- TestProductionRegression §8 the exact run-4f1e8395 state +""" + +import inspect +import logging +import random +import time +import types + +from src.agent.agent import Agent +from src.agent.message_log import MessageLog +from src.agent.simulation import SimulationEngine +from src.agent.state import ThreadState + + +def _settings(**kw): + base = dict( + cohort_isolation_enabled=False, + cohort_default_policy="open", + max_consecutive_reactive_turns=3, + turn_delay_seconds=0.0, + active_thread_threshold=12, + llm_rate_window_seconds=600, + llm_calls_per_load_per_window=8, + ) + base.update(kw) + return types.SimpleNamespace(**base) + + +def _patch(monkeypatch, **kw): + monkeypatch.setattr("src.agent.simulation.get_settings", lambda: _settings(**kw)) + + +def _engine(agent_ids, budget_cap=0): + agents = [ + Agent(agent_id=a, bot_name=f"{a.capitalize()}Bot", pi_name=f"PI {a}") + for a in agent_ids + ] + return SimulationEngine(agents=agents, slack_clients={}, budget_cap=budget_cap) + + +# Every awaitable the main loop calls once per tick before it selects an agent. +# All of them are I/O (Slack, DB, disk) and none of them affect selection, so a +# loop-level test stubs the lot and keeps only the scheduling behaviour. +_TICK_IO = ( + "_poll_slack_for_pi_messages", + "_poll_pi_dms", + "_poll_proposal_threads_for_pi", + "_poll_inbound_from_db", + "_poll_pi_dms_from_db", + "_sync_proposal_reviews_from_db", + "_sync_private_channels_from_db", + "_sync_roster_from_db", + "_flush_persisted", + "_flush_llm_logs", +) + + +def _drive_loop(eng, monkeypatch, *, stop_after=4): + """Run the REAL ``_run_main_loop`` with every per-tick I/O call stubbed out. + + Returns ``(sleeps, turns)``, both filled in as the loop runs. The engine is + stopped once ``stop_after`` events have been recorded, so a regression that + reinstates the old spin-or-break behaviour fails an assertion instead of + hanging the suite. + """ + async def _noop(*a, **kw): + return None + + for name in _TICK_IO: + monkeypatch.setattr(eng, name, _noop) + monkeypatch.setattr(eng, "_sync_profiles_from_disk", lambda *a, **kw: None) + + sleeps: list[int] = [] + turns: list[str] = [] + + def _budget(): + if len(sleeps) + len(turns) >= stop_after: + eng._running = False + + async def _sleep(delay): + sleeps.append(delay) + _budget() + + async def _run_turn(agent): + turns.append(agent.agent_id) + _budget() + return False + + monkeypatch.setattr(eng, "_sleep", _sleep) + monkeypatch.setattr(eng, "_run_turn", _run_turn) + eng._running = True + return sleeps, turns + + +def _add_threads(agent, n, *, status="active", pending=False, prefix="t"): + for i in range(n): + tid = f"{prefix}{i}" + agent.state.active_threads[tid] = ThreadState( + thread_id=tid, + channel="general", + other_agent_id=f"pi{i}", + status=status, + has_pending_reply=pending, + ) + + +class TestAgentLoad: + def test_idle_agent_has_load_one(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["hub"]) + assert eng._agent_load(eng.agents["hub"]) == 1 + + def test_load_counts_active_threads(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["hub"]) + _add_threads(eng.agents["hub"], 5) + assert eng._agent_load(eng.agents["hub"]) == 5 + + def test_non_active_threads_are_excluded(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["hub"]) + _add_threads(eng.agents["hub"], 3, status="active", prefix="a") + _add_threads(eng.agents["hub"], 4, status="closed", prefix="c") + _add_threads(eng.agents["hub"], 2, status="proposed", prefix="p") + assert eng._agent_load(eng.agents["hub"]) == 3 + + def test_load_is_clamped_at_active_thread_threshold(self, monkeypatch): + _patch(monkeypatch, active_thread_threshold=12) + eng = _engine(["hub"]) + _add_threads(eng.agents["hub"], 56) + assert eng._agent_load(eng.agents["hub"]) == 12 + + +class TestCallLedger: + def test_record_api_call_increments_both_counters(self): + a = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + a.record_api_call(now=100.0) + a.record_api_call(now=101.0) + assert a.api_call_count == 2 + assert list(a.state.call_times) == [100.0, 101.0] + + def test_record_api_call_defaults_to_wall_clock(self): + a = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + before = time.time() + a.record_api_call() + after = time.time() + assert a.api_call_count == 1 + assert before <= a.state.call_times[0] <= after + + def test_call_times_starts_empty(self): + a = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + assert len(a.state.call_times) == 0 + assert a.api_call_count == 0 + + +class TestRateLimiter: + def test_under_allowance_is_eligible(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(7): + a.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(a, 1010.0) is True + + def test_at_allowance_is_throttled(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(8): + a.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(a, 1010.0) is False + + def test_throttle_self_heals_as_the_window_slides(self, monkeypatch): + """The regression test for the permanent bench. A throttled agent MUST + become eligible again once its calls age out — this is the single + property the cumulative cap did not have.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8, + llm_rate_window_seconds=600) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(8): + a.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(a, 1010.0) is False + # 700s later every recorded call is outside the 600s window. + assert eng._within_rate_limit(a, 1710.0) is True + assert len(a.state.call_times) == 0 + + def test_allowance_scales_with_load(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8, + active_thread_threshold=12) + eng = _engine(["hub"]) + hub = eng.agents["hub"] + _add_threads(hub, 12) + for i in range(50): + hub.record_api_call(now=1000.0 + i) + # load 12 -> allowance 96, so 50 calls is fine for a hub... + assert eng._within_rate_limit(hub, 1060.0) is True + # ...but the identical ledger throttles a load-1 spoke. + spoke = Agent(agent_id="spoke", bot_name="SpokeBot", pi_name="PI spoke") + for i in range(50): + spoke.record_api_call(now=1000.0 + i) + assert eng._within_rate_limit(spoke, 1060.0) is False + + def test_role_override_beats_the_global_setting(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + monkeypatch.setattr( + "src.agent.simulation.load_role", + lambda name: types.SimpleNamespace(calls_per_load_per_window=2), + ) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + for i in range(3): + a.record_api_call(now=1000.0 + i) + assert eng._calls_per_load(a) == 2 + assert eng._within_rate_limit(a, 1010.0) is False + + def test_turn_eligible_requires_both_checks(self, monkeypatch): + """Legacy cumulative cap and the rate limiter compose with AND.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"], budget_cap=5) + a = eng.agents["spoke"] + # Rate limit fine (1 call), legacy cap blown (api_call_count 6 >= 5). + a.api_call_count = 6 + a.record_api_call(now=1000.0) + assert eng._turn_eligible(a, 1010.0) is False + + def test_turn_eligible_passes_when_both_pass(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"], budget_cap=0) + a = eng.agents["spoke"] + a.record_api_call(now=1000.0) + assert eng._turn_eligible(a, 1010.0) is True + + def test_throttle_flag_stays_fresh_while_the_legacy_cap_binds(self, monkeypatch): + """F5. The cap-first ordering in _turn_eligible is deliberate, but it used + to freeze `state.throttled`: with --budget armed the flag was whatever it + was when the cap first bit, so the agent's next real throttle transition + logged nothing. The window check now runs for its side effect while the + cap still decides eligibility. + """ + _patch(monkeypatch, llm_calls_per_load_per_window=2) + eng = _engine(["spoke"], budget_cap=5) + a = eng.agents["spoke"] + a.record_api_call(now=1000.0) + a.record_api_call(now=1001.0) # at the window allowance + a.api_call_count = 6 # ...and over the legacy cap + + assert eng._turn_eligible(a, 1010.0) is False + assert a.state.throttled is True + # 700s later the window has slid: the flag must clear even though the + # cap still benches the agent. Eligibility is unchanged either way. + assert eng._turn_eligible(a, 1710.0) is False + assert a.state.throttled is False + + def test_legacy_cap_still_decides_eligibility(self, monkeypatch): + """The F5 side-effect call must not have reordered the gates: an agent + well inside its rate window is STILL ineligible once the cap is blown.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["spoke"], budget_cap=5) + a = eng.agents["spoke"] + a.api_call_count = 5 + assert a.state.throttled is False # rate limit nowhere near + assert eng._turn_eligible(a, 1010.0) is False + + def test_throttle_transition_warns_once(self, monkeypatch, caplog): + _patch(monkeypatch, llm_calls_per_load_per_window=2) + eng = _engine(["spoke"]) + a = eng.agents["spoke"] + a.record_api_call(now=1000.0) + a.record_api_call(now=1001.0) + with caplog.at_level(logging.WARNING): + eng._within_rate_limit(a, 1010.0) + eng._within_rate_limit(a, 1011.0) + eng._within_rate_limit(a, 1012.0) + assert caplog.text.count("throttled") == 1 + + +class TestRestartRebuild: + def test_window_filter_selects_only_recent_calls(self, monkeypatch): + """Step 4b's cutoff arithmetic, isolated from the DB. + + The full DB round trip is covered by the integration suite; what matters + here is that the cutoff is `now - window` and that boundary rows are + included, since an off-by-one there silently re-creates the permanent + bench for anything on the edge. + """ + _patch(monkeypatch, llm_rate_window_seconds=600) + eng = _engine(["hub"]) + a = eng.agents["hub"] + now = 10_000.0 + # Simulate what step 4b loads: only rows at or after the cutoff. + cutoff = now - 600 + rows = [now - 1200, now - 700, now - 600, now - 100, now - 1] + a.state.call_times.extend(t for t in rows if t >= cutoff) + assert list(a.state.call_times) == [now - 600, now - 100, now - 1] + assert eng._within_rate_limit(a, now) is True + + def test_agent_whose_calls_all_predate_the_window_starts_unthrottled( + self, monkeypatch + ): + """The exact post-restart state that benched the hub: a large lifetime + count, but nothing inside the window.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8) + eng = _engine(["hub"]) + a = eng.agents["hub"] + a.api_call_count = 42 # rebuilt by step 4, lifetime + # step 4b found no rows inside the window + assert eng._within_rate_limit(a, 10_000.0) is True + assert eng._turn_eligible(a, 10_000.0) is True + + +class TestScheduler: + def test_proactive_weight_scales_with_load(self, monkeypatch): + """A load-12 hub against 12 load-1 spokes, all equally stale, should take + ~12/(12+12) = 50% of proactive draws. Under the old agent-fair weighting + it took 1/13 = 7.7%.""" + _patch(monkeypatch, active_thread_threshold=12) + random.seed(20260806) + ids = ["hub"] + [f"pi{i}" for i in range(12)] + eng = _engine(ids) + _add_threads(eng.agents["hub"], 12) + now = time.time() + for a in eng.agents.values(): + a.state.last_selected = now - 100.0 + + picks = [eng._select_agent().agent_id for _ in range(2000)] + share = picks.count("hub") / 2000 + assert 0.42 < share < 0.58, f"hub share {share:.3f} not load-proportional" + + def test_reactive_tiebreak_no_longer_penalises_the_busy_agent( + self, monkeypatch + ): + """The hub is selected often, so its last_selected is always recent. Under + min(last_selected) it lost every tiebreak to a long-idle spoke — it was + penalised precisely for being busy. Weighted by load, it wins.""" + _patch(monkeypatch, active_thread_threshold=12) + eng = _engine(["hub", "spoke"]) + now = time.time() + _add_threads(eng.agents["hub"], 12, pending=True) + _add_threads(eng.agents["spoke"], 1, pending=True, prefix="s") + eng.agents["hub"].state.last_selected = now - 10.0 # 10 * 12 = 120 + eng.agents["spoke"].state.last_selected = now - 60.0 # 60 * 1 = 60 + + assert eng._select_agent().agent_id == "hub" + + def test_reactive_tier_still_prefers_a_genuinely_starved_spoke( + self, monkeypatch + ): + """The load weighting must not become a blank cheque: a spoke that has + waited long enough still outranks the hub.""" + _patch(monkeypatch, active_thread_threshold=12) + eng = _engine(["hub", "spoke"]) + now = time.time() + _add_threads(eng.agents["hub"], 2, pending=True) + _add_threads(eng.agents["spoke"], 1, pending=True, prefix="s") + eng.agents["hub"].state.last_selected = now - 10.0 # 10 * 2 = 20 + eng.agents["spoke"].state.last_selected = now - 600.0 # 600 * 1 = 600 + + assert eng._select_agent().agent_id == "spoke" + + def test_throttled_hub_is_not_selected(self, monkeypatch): + _patch(monkeypatch, llm_calls_per_load_per_window=1, + active_thread_threshold=12) + eng = _engine(["hub", "spoke"]) + now = time.time() + hub = eng.agents["hub"] + _add_threads(hub, 1) + hub.record_api_call(now=now) + for _ in range(50): + assert eng._select_agent().agent_id == "spoke" + + +class TestBudgetDeprecation: + def test_default_budget_is_off(self): + """The default must be 0 (off). A nonzero default is what silently armed + the legacy cap on every run.""" + from src.agent.main import main + + default = inspect.signature(main).parameters["budget"].default + assert default.default == 0 + + def test_help_text_marks_the_flag_deprecated(self): + from src.agent.main import main + + help_text = inspect.signature(main).parameters["budget"].default.help + assert "DEPRECATED" in help_text + + def test_engine_constructor_default_matches_the_cli_default(self): + """F3. The constructor default was 50 while the CLI default was 0, so + every caller that omitted the kwarg — tests, backfill scripts — silently + armed the deprecated permanent cap.""" + default = inspect.signature(SimulationEngine.__init__).parameters[ + "budget_cap" + ].default + assert default == 0 + + def test_engine_without_a_budget_kwarg_caps_nobody(self, monkeypatch): + _patch(monkeypatch) + agent = Agent(agent_id="hub", bot_name="HubBot", pi_name="PI hub") + agent.api_call_count = 10_000 + eng = SimulationEngine(agents=[agent], slack_clients={}) + assert eng._agent_within_budget(agent) is True + assert eng._turn_eligible(agent, time.time()) is True + + +class TestStallIsTransient: + """F1. `_select_agent()` returning None used to break the main loop. + + Rate limiting and the per-agent `turn_delay_seconds` cooldown both lapse with + time, so breaking on them converts "one agent is benched for a while" into + "the run exits and stays exited" — strictly worse than the bug the branch + fixes, and reachable on any roster whose aggregate demand meets its aggregate + allowance (7 token-holding agents at 8 calls/600s does it in minutes). + """ + + def test_predicate_is_not_terminal_when_the_cap_is_off(self, monkeypatch): + _patch(monkeypatch) + eng = _engine(["a", "b"], budget_cap=0) + for a in eng.agents.values(): + a.api_call_count = 10_000 + assert eng._terminal_stall_reason() is None + + def test_predicate_is_not_terminal_while_one_agent_is_under_the_cap( + self, monkeypatch + ): + _patch(monkeypatch) + eng = _engine(["a", "b"], budget_cap=5) + eng.agents["a"].api_call_count = 99 + eng.agents["b"].api_call_count = 1 + assert eng._terminal_stall_reason() is None + + def test_predicate_is_terminal_when_every_agent_is_over_the_cap( + self, monkeypatch + ): + _patch(monkeypatch) + eng = _engine(["a", "b"], budget_cap=5) + for a in eng.agents.values(): + a.api_call_count = 5 + assert "legacy --budget cap (5)" in eng._terminal_stall_reason() + + def test_predicate_is_terminal_on_an_empty_roster(self, monkeypatch): + _patch(monkeypatch) + eng = _engine([], budget_cap=0) + assert eng._terminal_stall_reason() == "the roster is empty" + + async def test_fully_throttled_roster_backs_off_instead_of_stopping( + self, monkeypatch, caplog + ): + """THE regression test for F1: every agent throttled, legacy cap off. + + The loop must keep ticking with a growing backoff, never break, and never + spin (each empty tick costs a `_sleep`). + """ + _patch(monkeypatch, llm_calls_per_load_per_window=1) + eng = _engine(["a", "b"], budget_cap=0) + now = time.time() + for a in eng.agents.values(): + a.record_api_call(now=now) # allowance is 1 * load 1 + assert eng._select_agent() is None + + sleeps, turns = _drive_loop(eng, monkeypatch, stop_after=4) + with caplog.at_level(logging.INFO): + await eng._run_main_loop() + + assert turns == [] + assert sleeps == [5, 5, 5, 15], "idle backoff must apply and grow" + assert "Stopping" not in caplog.text + assert "retrying" in caplog.text + + async def test_mixed_stall_with_the_cap_armed_is_still_transient( + self, monkeypatch, caplog + ): + """Cap armed, one agent over it, the other merely throttled. Nothing is + selectable right now, but the second agent recovers as the window slides, + so the run must not end.""" + _patch(monkeypatch, llm_calls_per_load_per_window=1) + eng = _engine(["over", "throttled"], budget_cap=5) + eng.agents["over"].api_call_count = 99 + eng.agents["throttled"].record_api_call(now=time.time()) + assert eng._select_agent() is None + + sleeps, turns = _drive_loop(eng, monkeypatch, stop_after=2) + with caplog.at_level(logging.INFO): + await eng._run_main_loop() + + assert turns == [] + assert sleeps == [5, 5] + assert "Stopping" not in caplog.text + + async def test_every_agent_over_the_legacy_cap_stops_the_loop( + self, monkeypatch, caplog + ): + """The one genuinely permanent case: `api_call_count` only grows within a + process and `_rebuild_state_from_db` restores it, so nothing recovers.""" + _patch(monkeypatch) + eng = _engine(["a", "b"], budget_cap=5) + for a in eng.agents.values(): + a.api_call_count = 5 + + sleeps, turns = _drive_loop(eng, monkeypatch, stop_after=4) + with caplog.at_level(logging.INFO): + await eng._run_main_loop() + + assert (sleeps, turns) == ([], []), "terminal stall must break at once" + assert "over the legacy --budget cap (5). Stopping." in caplog.text + + async def test_a_recovered_agent_gets_its_turn(self, monkeypatch): + """The backoff is not a dead end: once the ledger clears, the very next + tick selects the agent it was waiting for.""" + _patch(monkeypatch, llm_calls_per_load_per_window=1) + eng = _engine(["a"], budget_cap=0) + eng.agents["a"].record_api_call(now=time.time()) + + sleeps, turns = _drive_loop(eng, monkeypatch, stop_after=3) + # The first stall clears the ledger the way an expiring window would. + real_sleep = eng._sleep + + async def _sleep(delay): + eng.agents["a"].state.call_times.clear() + await real_sleep(delay) + + monkeypatch.setattr(eng, "_sleep", _sleep) + await eng._run_main_loop() + + assert turns and turns[0] == "a" + + async def test_stop_signal_still_ends_a_stalled_loop(self, monkeypatch): + """max_runtime / SIGTERM must still end the run: `_sleep` returns early + once `_stop_event` is set, and the loop condition then fails.""" + _patch(monkeypatch, llm_calls_per_load_per_window=1) + eng = _engine(["a"], budget_cap=0) + eng.agents["a"].record_api_call(now=time.time()) + + sleeps, turns = _drive_loop(eng, monkeypatch, stop_after=50) + real_sleep = eng._sleep + + async def _sleep(delay): + eng.request_stop() + await real_sleep(delay) + + monkeypatch.setattr(eng, "_sleep", _sleep) + await eng._run_main_loop() + + assert turns == [] + assert len(sleeps) == 1 + + +class TestPIHandlerAccounting: + """F2. `pi_handler` logged llm_call_logs rows under a real agent_id without + touching either counter, so those calls were invisible to the live limiter + but restored into `call_times` by step 4b — throttling an agent on turn 0 of + a resumed run for calls it never appeared to make. + """ + + def _handler(self, monkeypatch, response="answer", raises=False): + from src.agent import pi_handler as ph + + agent = Agent(agent_id="su", bot_name="SuBot", pi_name="Andrew Su") + handler = ph.PIHandler( + agents={"su": agent}, + slack_clients={}, + pi_slack_id_to_agent_ids={"U1": ["su"]}, + message_log=MessageLog(), + ) + + async def _fake_llm(**kwargs): + if raises: + raise RuntimeError("anthropic is down") + return response + + async def _fake_dm(*a, **kw): + return None + + monkeypatch.setattr(ph, "generate_agent_response", _fake_llm) + monkeypatch.setattr(handler, "_send_dm", _fake_dm) + monkeypatch.setattr(agent, "update_private_profile", lambda text: None) + return handler, agent + + async def test_pi_question_is_recorded_against_the_agent(self, monkeypatch): + handler, agent = self._handler(monkeypatch) + await handler._handle_question("su", "U1", "how many threads do you have?") + assert agent.api_call_count == 1 + assert len(agent.state.call_times) == 1 + + async def test_profile_rewrite_is_recorded_against_the_agent(self, monkeypatch): + handler, agent = self._handler( + monkeypatch, response="<profile>new</profile><changes>x</changes>", + ) + await handler._handle_standing_instruction("su", "U1", "always cite DOIs") + assert agent.api_call_count == 1 + assert len(agent.state.call_times) == 1 + + async def test_a_failed_call_is_not_recorded(self, monkeypatch): + handler, agent = self._handler(monkeypatch, raises=True) + await handler._handle_question("su", "U1", "anything") + assert agent.api_call_count == 0 + assert len(agent.state.call_times) == 0 + + async def test_dm_classification_is_not_attributed_to_the_agent( + self, monkeypatch + ): + """`_classify_dm` logs under the synthetic agent_id "pi_handler", so the + restart rebuild attributes it to nobody. Counting it live would make the + in-process ledger disagree in the other direction.""" + handler, agent = self._handler(monkeypatch, response='{"category": "question"}') + await handler._classify_dm("what are you working on?") + assert agent.api_call_count == 0 + assert len(agent.state.call_times) == 0 + + async def test_a_pi_dm_burst_shows_up_in_the_live_rate_limiter( + self, monkeypatch + ): + """The end-to-end point of F2: ten PI questions must throttle the agent + NOW, exactly as they would after a restart rebuilt them from the DB.""" + _patch(monkeypatch, llm_calls_per_load_per_window=8) + handler, agent = self._handler(monkeypatch) + eng = SimulationEngine(agents=[agent], slack_clients={}) + for _ in range(10): + await handler._handle_question("su", "U1", "status?") + assert agent.api_call_count == 10 + assert eng._within_rate_limit(agent, time.time()) is False + + +class TestRateSettingGuards: + """F4. `roles.py` rejects a non-positive per-role override; the global + settings had no such guard. `llm_calls_per_load_per_window=0` makes + `len(times) < 0` false for everyone, so — now that a stall no longer ends the + run — a typo buys a silently, permanently idle simulation. + """ + + def _settings_obj(self, **kw): + from src.config import Settings + + return Settings(_env_file=None, environment="development", **kw) + + def test_zero_calls_per_load_is_clamped_to_the_default(self, caplog): + with caplog.at_level(logging.WARNING, logger="src.config"): + s = self._settings_obj(llm_calls_per_load_per_window=0) + assert s.llm_calls_per_load_per_window == 8 + assert "LLM_CALLS_PER_LOAD_PER_WINDOW" in caplog.text + + def test_negative_window_is_clamped_to_the_default(self, caplog): + with caplog.at_level(logging.WARNING, logger="src.config"): + s = self._settings_obj(llm_rate_window_seconds=-1) + assert s.llm_rate_window_seconds == 600 + assert "LLM_RATE_WINDOW_SECONDS" in caplog.text + + def test_valid_overrides_are_left_alone(self, caplog): + with caplog.at_level(logging.WARNING, logger="src.config"): + s = self._settings_obj( + llm_calls_per_load_per_window=2, llm_rate_window_seconds=30, + ) + assert (s.llm_calls_per_load_per_window, s.llm_rate_window_seconds) == (2, 30) + assert "must be a positive int" not in caplog.text + + def test_a_clamped_setting_leaves_agents_selectable(self, monkeypatch): + """The behavioural consequence: with the guard, a 0 in the environment + degrades to the default allowance instead of benching the whole roster.""" + s = self._settings_obj(llm_calls_per_load_per_window=0) + monkeypatch.setattr("src.agent.simulation.get_settings", lambda: s) + eng = _engine(["a"]) + assert eng._turn_eligible(eng.agents["a"], time.time()) is True + + +class TestProductionRegression: + """Reconstructs the exact state of run 4f1e8395 (2026-08-05), in which the + blackbird hub took 0 of 161 turns while 56 spokes took 3-5 each. + + Measured then: hub 42 LLM calls, next-busiest agent 9, cap 40. + """ + + def _star(self, monkeypatch, budget_cap, **kw): + _patch(monkeypatch, active_thread_threshold=12, **kw) + ids = ["blackbird"] + [f"pi{i}" for i in range(56)] + eng = _engine(ids, budget_cap=budget_cap) + eng.agents["blackbird"].api_call_count = 42 + for i in range(56): + eng.agents[f"pi{i}"].api_call_count = 8 + return eng + + def test_fixed_hub_is_selectable_after_restart(self, monkeypatch): + """Case 1 — THE FIX. New default (budget_cap=0), lifetime count 42, but + nothing inside the window because step 4b found no recent rows.""" + eng = self._star(monkeypatch, budget_cap=0) + hub = eng.agents["blackbird"] + now = time.time() + assert eng._turn_eligible(hub, now) is True + + random.seed(20260806) + picks = [eng._select_agent().agent_id for _ in range(2000)] + assert picks.count("blackbird") > 0, "hub still benched — the fix failed" + + def test_throttling_is_still_real_but_temporary(self, monkeypatch): + """Case 2 — the limiter has not been defanged. A load-1 hub that burns + its allowance inside the window IS throttled, then recovers.""" + eng = self._star(monkeypatch, budget_cap=0, + llm_calls_per_load_per_window=8, + llm_rate_window_seconds=600) + hub = eng.agents["blackbird"] + base = 10_000.0 + for i in range(8): + hub.record_api_call(now=base + i) + assert eng._turn_eligible(hub, base + 10) is False + assert eng._turn_eligible(hub, base + 700) is True + + def test_legacy_budget_flag_still_benches_the_hub(self, monkeypatch): + """Case 3 — the compat path, pinned honestly. --budget 40 was NOT made + safe; it was deprecated and defaulted off. If someone passes it, the old + behaviour is exactly what they get.""" + eng = self._star(monkeypatch, budget_cap=40) + hub = eng.agents["blackbird"] + now = time.time() + assert eng._turn_eligible(hub, now) is False + picks = [eng._select_agent().agent_id for _ in range(500)] + assert "blackbird" not in picks diff --git a/tests/unit/test_ids.py b/tests/unit/test_ids.py new file mode 100644 index 0000000..2d1d670 --- /dev/null +++ b/tests/unit/test_ids.py @@ -0,0 +1,129 @@ +"""Tests for canonical ts-shaped id minting (src/agent/ids.py).""" + +import pytest + +from src.agent.ids import ( + WRITER_ENGINE, + WRITER_GRANTBOT, + WRITER_SLOT_MODULUS, + WRITER_WEB, + TsMinter, + default_writer_id, + mint_local_ts, + set_default_writer_id, +) + + +class TestTsMinter: + def test_monotonic_unique_and_float_ordered(self): + m = TsMinter() + ids = [m.mint() for _ in range(2000)] + # Distinct strings (DB uniqueness) ... + assert len(set(ids)) == len(ids) + # ... and strictly increasing once parsed to float (posted_at ordering). + floats = [float(x) for x in ids] + assert all(b > a for a, b in zip(floats, floats[1:], strict=False)) + + def test_ts_shape_is_seconds_dot_six_microsecond_digits(self): + m = TsMinter() + secs, _, micros = m.mint().partition(".") + assert secs.isdigit() + assert len(micros) == 6 and micros.isdigit() + + def test_seed_floor_pushes_ids_after_a_future_high_water_mark(self): + import time + + m = TsMinter() + future = time.time() + 3600 + m.seed_floor(future) + assert float(m.mint()) > future + + def test_seed_floor_never_lowers_the_mark(self): + m = TsMinter() + first = m.mint() + m.seed_floor(0.0) # far below the current wall clock — must be ignored + assert m.mint() > first + + def test_seed_floor_from_another_writers_id_still_sorts_after(self): + # Restored history can be a *different* writer's id, which sits in a + # different residue class — the floor must clear it regardless. + other = TsMinter(WRITER_GRANTBOT) + history = float(other.mint()) + m = TsMinter(WRITER_ENGINE) + m.seed_floor(history) + assert float(m.mint()) > history + + +class TestWriterSlots: + """R1: two processes minting in the same microsecond must not collide.""" + + def test_ids_carry_the_writer_id_in_the_low_digits(self): + for writer_id in (WRITER_ENGINE, WRITER_WEB, WRITER_GRANTBOT): + m = TsMinter(writer_id) + us = round(float(m.mint()) * 1_000_000) + # float() round-trips only ~0.25us at this magnitude, so compare the + # residue on the integer microseconds parsed from the string parts. + secs, _, micros = m.mint().partition(".") + assert int(micros) % WRITER_SLOT_MODULUS == writer_id + assert us % WRITER_SLOT_MODULUS == writer_id + + def test_concurrent_writers_never_produce_the_same_id(self): + # The exact scenario the DB constraint used to catch by dropping a + # message: independent minters running flat out at the same instant. + engine = TsMinter(WRITER_ENGINE) + web = TsMinter(WRITER_WEB) + grantbot = TsMinter(WRITER_GRANTBOT) + ids = [] + for _ in range(500): + ids.append(engine.mint()) + ids.append(web.mint()) + ids.append(grantbot.mint()) + assert len(set(ids)) == len(ids) + + def test_same_wall_clock_instant_still_disjoint(self, monkeypatch): + # Pin the clock so every mint sees the identical microsecond — the + # per-process counters alone would hand out the same id here. + import time as time_mod + + monkeypatch.setattr(time_mod, "time_ns", lambda: 1_800_000_000_000_000_000) + engine = TsMinter(WRITER_ENGINE) + web = TsMinter(WRITER_WEB) + assert engine.mint() != web.mint() + + def test_rejects_an_out_of_range_writer_id(self): + with pytest.raises(ValueError): + TsMinter(WRITER_SLOT_MODULUS) + with pytest.raises(ValueError): + TsMinter(-1) + + +class TestModuleDefaultMinter: + def test_mint_local_ts_is_monotonic_across_calls(self): + # The process-wide minter used by the web PI inbox and GrantBot. + a = mint_local_ts() + b = mint_local_ts() + assert b > a + assert a != b + + def test_set_default_writer_id_switches_residue_class(self): + original = default_writer_id() + try: + set_default_writer_id(WRITER_GRANTBOT) + assert default_writer_id() == WRITER_GRANTBOT + _, _, micros = mint_local_ts().partition(".") + assert int(micros) % WRITER_SLOT_MODULUS == WRITER_GRANTBOT + finally: + set_default_writer_id(original) + + def test_set_default_writer_id_keeps_ids_monotonic_across_the_swap(self): + original = default_writer_id() + try: + set_default_writer_id(WRITER_WEB) + before = mint_local_ts() + # A fresh counter would restart from the current microsecond and + # could reuse the slot just consumed; the high-water mark carries. + set_default_writer_id(WRITER_GRANTBOT) + after = mint_local_ts() + assert float(after) > float(before) + finally: + set_default_writer_id(original) diff --git a/tests/unit/test_lab_directory_ordering.py b/tests/unit/test_lab_directory_ordering.py new file mode 100644 index 0000000..a0d9300 --- /dev/null +++ b/tests/unit/test_lab_directory_ordering.py @@ -0,0 +1,132 @@ +"""The lab directory must be gate-scoped in the order production builds it. + +src/agent/simulation.py's _build_lab_directories filters the directory by allowed_sender_ids, but +start() built it at :508 and only computed the gate at :533 — so every gate was +still None and the filter no-opped. On a stable roster it was never rebuilt. + +Measured in production: gill's phase-5 system prompt named 51 labs it could not +reach, "Blackbird" (its one reachable partner) appeared nowhere, and the +directory was 69% of a 67 KB prompt. + +The pre-existing test (tests/unit/test_simulation_logic.py) sets the gates by +hand BEFORE calling the builder, which is why it passed throughout. +""" +from src.agent.agent import Agent +from src.agent.simulation import SimulationEngine + + +def _agent(aid: str, pub: str, role: str = "pi_lab") -> Agent: + a = Agent(aid, f"{aid.capitalize()}Bot", f"{aid.upper()} PI", role=role) + a._public_profile = f"# {aid} Lab\n\n## Recent Publications\n- {pub}\n" + return a + + +def test_a_fresh_agents_gate_is_none(): + """The precondition that made the ordering matter.""" + assert _agent("a", "paper A").allowed_sender_ids is None + + +def test_directory_is_gate_scoped_after_the_gate_is_applied(): + """Whatever the internal ordering, once gates exist the directory must agree + with them. This is the invariant; it does not care how it is achieved.""" + a, b, c = _agent("a", "paper A"), _agent("b", "paper B"), _agent("c", "paper C") + eng = SimulationEngine(agents=[a, b, c], slack_clients={}) + + a.allowed_sender_ids = {"a", "b"} + b.allowed_sender_ids = {"a", "b"} + c.allowed_sender_ids = {"c"} + eng.refresh_lab_directories() + + assert "paper B" in (a._lab_directory or "") + assert "paper C" not in (a._lab_directory or "") + assert c._lab_directory is None + + +def test_refresh_is_idempotent(): + a, b = _agent("a", "paper A"), _agent("b", "paper B") + a.allowed_sender_ids = {"a", "b"} + b.allowed_sender_ids = {"a", "b"} + eng = SimulationEngine(agents=[a, b], slack_clients={}) + eng.refresh_lab_directories() + first = a._lab_directory + eng.refresh_lab_directories() + assert a._lab_directory == first + + +def test_tightening_a_gate_then_refreshing_removes_the_stale_lab(): + """The gate-change rebuild: a topology edit mid-run must not leave an agent + primed with a lab it can no longer reach.""" + a, b = _agent("a", "paper A"), _agent("b", "paper B") + a.allowed_sender_ids = {"a", "b"} + b.allowed_sender_ids = {"a", "b"} + eng = SimulationEngine(agents=[a, b], slack_clients={}) + eng.refresh_lab_directories() + assert "paper B" in (a._lab_directory or "") + + a.allowed_sender_ids = {"a"} + eng.refresh_lab_directories() + assert a._lab_directory is None + + +def test_gate_off_still_lists_every_other_lab(): + """Mesh behaviour is unchanged: gate None means no filtering.""" + a, b = _agent("a", "paper A"), _agent("b", "paper B") + eng = SimulationEngine(agents=[a, b], slack_clients={}) + eng.refresh_lab_directories() + assert "paper B" in (a._lab_directory or "") + + +# --- the two that pin the actual bug ---------------------------------------- +# +# Everything above calls refresh_lab_directories() by hand, which is what the +# PRE-EXISTING test did — and it is why the bug survived. The predicate was +# never broken; the ORDER was. These two guard the order. + + +def test_start_computes_the_gate_before_it_builds_the_directory(): + """A source assertion, deliberately. + + start() does too much I/O to drive in a unit test, and the failure mode is a + reordering — exactly the edit a future refactor makes silently, and exactly + what no behavioural test in this file would catch. Reading the source is + crude but it is the thing that was actually wrong. + """ + import inspect + + src = inspect.getsource(SimulationEngine.start) + gate = src.index("_recompute_allowed_sender_ids") + build = src.index("refresh_lab_directories") + assert gate < build, ( + "start() builds the lab directory before computing the cohort gate; " + "every agent's allowed_sender_ids is still None at that point, so the " + "filter inside _build_lab_directories no-ops" + ) + + +async def test_recompute_refreshes_the_directory_when_it_disables_the_gate(monkeypatch): + """The durable half of the fix, driven through the real method. + + _recompute_allowed_sender_ids owns the gate, so it must own the directory + derived from it. The isolation-disabled path is the cheap way to prove that + without a database: it sets every gate to None, and the directory must widen + to match instead of staying scoped to a gate that no longer applies. + """ + import types + + monkeypatch.setattr( + "src.agent.simulation.get_settings", + lambda: types.SimpleNamespace(cohort_isolation_enabled=False), + ) + a, b = _agent("a", "paper A"), _agent("b", "paper B") + a.allowed_sender_ids = {"a"} # isolated under the old topology + b.allowed_sender_ids = {"b"} + eng = SimulationEngine(agents=[a, b], slack_clients={}) + eng.refresh_lab_directories() + assert a._lab_directory is None # correctly empty while isolated + + await eng._recompute_allowed_sender_ids() + + assert a.allowed_sender_ids is None + assert "paper B" in (a._lab_directory or ""), ( + "the gate was disabled but the directory still reflects the old one" + ) diff --git a/tests/unit/test_llm_service.py b/tests/unit/test_llm_service.py index 5e53168..84b467f 100644 --- a/tests/unit/test_llm_service.py +++ b/tests/unit/test_llm_service.py @@ -34,6 +34,67 @@ async def test_generate_agent_response_retries_once_on_max_tokens(monkeypatch): assert fake.calls[1]["max_tokens"] == 2000 # retried at 2x the original cap +async def test_generate_agent_response_calls_on_retry_when_it_actually_retries(monkeypatch): + """A retried turn is two real, billed API calls. A caller booking one call + per turn against the sliding-window rate limiter (agent.record_api_call) + needs a hook to book the second one too, or the limiter undercounts + exactly the agent it exists to pace (Finding A1).""" + fake = FakeAnthropic( + [ + text_response("truncated...", stop_reason="max_tokens"), + text_response("full answer"), + ] + ) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + retry_calls = [] + out = await llm.generate_agent_response( + "sys", [{"role": "user", "content": "hi"}], max_tokens=1000, + on_retry=lambda: retry_calls.append(1), + ) + assert out == "full answer" + assert retry_calls == [1] + + +async def test_generate_agent_response_does_not_call_on_retry_without_truncation(monkeypatch): + fake = FakeAnthropic([text_response("full answer")]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + retry_calls = [] + out = await llm.generate_agent_response( + "sys", [{"role": "user", "content": "hi"}], max_tokens=1000, + on_retry=lambda: retry_calls.append(1), + ) + assert out == "full answer" + assert retry_calls == [] + + +async def test_generate_agent_response_logs_loudly_when_retry_still_truncates(monkeypatch, caplog): + """Before this fix, a still-truncated retry's stop_reason was never + re-checked, so a phase-5 assessment whose 15-line <assessment_json> + sidecar is emitted last could silently lose its verdict with no trace in + the logs. The (still truncated) text is still returned — never swallowed — + but the truncation must be loud and identifiable (Finding A1).""" + fake = FakeAnthropic( + [ + text_response("first truncated...", stop_reason="max_tokens"), + text_response("still truncated...", stop_reason="max_tokens"), + ] + ) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + out = await llm.generate_agent_response( + "sys", [{"role": "user", "content": "hi"}], max_tokens=1000, + log_meta={"agent_id": "blackbird", "phase": "new_post"}, + ) + + assert out == "still truncated..." # best-available text, not swallowed + assert len(fake.calls) == 2 # one retry only — no second retry added + assert "still truncated after 2x max_tokens retry" in caplog.text + assert "agent=blackbird" in caplog.text + assert "phase=new_post" in caplog.text + + async def test_generate_agent_response_empty_content_returns_blank(monkeypatch): fake = FakeAnthropic([empty_response()]) monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) diff --git a/tests/unit/test_message_log.py b/tests/unit/test_message_log.py index 2e055c4..42136df 100644 --- a/tests/unit/test_message_log.py +++ b/tests/unit/test_message_log.py @@ -137,6 +137,39 @@ def test_empty_thread(self, log): history = log.get_thread_history("999") assert history == [] + def test_orders_replies_by_posted_at_not_insertion_order(self, log): + # A reply ingested late by the DB poller or the Slack reconcile is + # appended after entries that came *after* it in real time. Insertion + # order would hand the LLM a scrambled thread. + log.append(_post("1", "general", "su", "SuBot", "Root")) + log.append(_post("30", "general", "wiseman", "WisemanBot", "third", thread_ts="1")) + log.append(_post("20", "general", "su", "SuBot", "second", thread_ts="1")) + log.append(_post("10", "general", "wiseman", "WisemanBot", "first", thread_ts="1")) + + history = log.get_thread_history("1") + assert [e.content for e in history] == ["Root", "first", "second", "third"] + + def test_root_stays_first_even_if_a_reply_predates_it(self, log): + # A writer whose clock runs behind can stamp a reply below the root's + # posted_at; the root is still the thread's parent. + log.append(_post("100", "general", "su", "SuBot", "Root")) + log.append(_post("50", "general", "wiseman", "WisemanBot", "skewed", thread_ts="100")) + log.append(_post("200", "general", "su", "SuBot", "later", thread_ts="100")) + + history = log.get_thread_history("100") + assert [e.content for e in history] == ["Root", "skewed", "later"] + + def test_equal_posted_at_keeps_insertion_order(self, log): + # Stable sort: nothing reshuffles when timestamps tie. + log.append(_post("1", "general", "su", "SuBot", "Root")) + for i, content in enumerate(("a", "b", "c")): + entry = _post(f"{i + 2}", "general", "wiseman", "WisemanBot", content, thread_ts="1") + entry.posted_at = 5.0 + log.append(entry) + + history = log.get_thread_history("1") + assert [e.content for e in history] == ["Root", "a", "b", "c"] + # --------------------------------------------------------------- # get_tags_for_agent @@ -208,3 +241,87 @@ def test_skips_human_messages(self, log): ) log.append(human) assert log.get_last_bot_sender_in_channel("priv-x") == "su" + + +# --------------------------------------------------------------- +# Idempotent append + persist callback (DB-primary store) +# --------------------------------------------------------------- + +class TestAppendIdempotencyAndPersist: + def test_append_returns_true_then_false_on_duplicate_ts(self, log): + assert log.append(_post("1", "general", "su", "SuBot", "first")) is True + # Same ts: skipped, returns False, no duplicate stored. + assert log.append(_post("1", "general", "su", "SuBot", "dup")) is False + assert len(log) == 1 + # The original content is retained (the duplicate is dropped). + assert log.get_entry("1").content == "first" + + def test_persist_callback_fires_once_per_new_append(self, log): + seen = [] + log.set_persist_callback(lambda e: seen.append(e.ts)) + log.append(_post("1", "general", "su", "SuBot", "a")) + log.append(_post("1", "general", "su", "SuBot", "a-dup")) # skipped + log.append(_post("2", "general", "wiseman", "WisemanBot", "b")) + assert seen == ["1", "2"] + + def test_load_entry_bypasses_callback(self, log): + seen = [] + log.set_persist_callback(lambda e: seen.append(e.ts)) + log.load_entry(_post("1", "general", "su", "SuBot", "restored")) + assert len(log) == 1 + assert seen == [] # rebuild path must not re-persist + # Still idempotent on ts. + log.load_entry(_post("1", "general", "su", "SuBot", "again")) + assert len(log) == 1 + + +# --------------------------------------------------------------- +# Insertion order is not time order: the DB inbound poller and the Slack +# reconcile append entries whose posted_at predates what is already stored, so +# every "most recent" query must key on posted_at, not on the tail of _entries. +# --------------------------------------------------------------- + +class TestOrderingIsByPostedAtNotInsertion: + def test_latest_timestamp_is_the_max_not_the_last_appended(self, log): + log.append(_post("100", "general", "su", "SuBot", "newest")) + # Ingested afterwards, but older — a tail read would move the cursor back. + log.append(_post("40", "general", "wiseman", "WisemanBot", "older, late")) + assert log.latest_timestamp == 100.0 + + def test_latest_timestamp_is_zero_on_an_empty_log(self, log): + assert log.latest_timestamp == 0.0 + + def test_latest_timestamp_counts_restored_entries(self, log): + log.load_entry(_post("70", "general", "su", "SuBot", "restored")) + assert log.latest_timestamp == 70.0 + + def test_agent_posts_slice_keeps_the_newest_by_posted_at(self, log): + # Newest post first, then two older ones ingested late. With limit=2 an + # insertion-order slice would drop "newest" — the post the Phase 5 dedup + # context and the daily cap most need to see. + log.append(_post("300", "general", "su", "SuBot", "newest")) + log.append(_post("100", "general", "su", "SuBot", "old-a")) + log.append(_post("200", "general", "su", "SuBot", "old-b")) + got = log.get_agent_top_level_posts("su", limit=2) + assert [e.content for e in got] == ["old-b", "newest"] # oldest first + + def test_agent_posts_still_exclude_replies_and_other_agents(self, log): + log.append(_post("10", "general", "su", "SuBot", "root")) + log.append(_post("20", "general", "su", "SuBot", "reply", thread_ts="10")) + log.append(_post("30", "general", "wiseman", "WisemanBot", "other")) + assert [e.content for e in log.get_agent_top_level_posts("su")] == ["root"] + + def test_last_bot_sender_ignores_a_late_appended_older_message(self, log): + log.append(_post("10", "priv-x", "wiseman", "WisemanBot", "first")) + log.append(_post("20", "priv-x", "su", "SuBot", "second — the real latest")) + # Reconcile pulls in a message that predates both; scanning the log + # backwards would name wiseman the last poster and hand su another turn. + log.append(_post("5", "priv-x", "wiseman", "WisemanBot", "older, late")) + assert log.get_last_bot_sender_in_channel("priv-x") == "su" + + def test_last_bot_sender_breaks_posted_at_ties_by_insertion(self, log): + log.append(_post("10", "priv-x", "wiseman", "WisemanBot", "a")) + tie = _post("10", "priv-x", "su", "SuBot", "b") + tie.ts = "10-b" # distinct id, identical posted_at + log.append(tie) + assert log.get_last_bot_sender_in_channel("priv-x") == "su" diff --git a/tests/unit/test_migration_checks.py b/tests/unit/test_migration_checks.py new file mode 100644 index 0000000..f94e05f --- /dev/null +++ b/tests/unit/test_migration_checks.py @@ -0,0 +1,1350 @@ +"""Pure-logic tests for scripts/migrate/preflight.py and scripts/migrate/postflight.py. + +No database and no Docker: everything here exercises the decision logic — exit-code +mapping, threshold decisions, query builders, the backup verdict table, the row-count +comparison and the alembic/env.py harness analyser. The DB-facing half of both scripts is +covered against throwaway ``pf_t*`` Postgres databases (see the report for that change). + +Both scripts are scripts, not an importable package (there is no ``__init__.py`` anywhere +under ``scripts/``), so they are loaded by path. Registering the module in ``sys.modules`` +BEFORE ``exec_module`` is load-bearing rather than tidiness: ``@dataclass`` resolves its +annotations through ``sys.modules[cls.__module__]``, and preflight.py defines three +dataclasses. + +Three tests here are regression guards for bugs that were real, were found by testing +against live fixtures, and would each have made a check fail OPEN — the worst possible +failure mode for a safety gate: + +* ``test_existing_object_names_casts_relkind_to_text`` — ``pg_class.relkind`` is + Postgres' internal ``"char"`` type and asyncpg decodes it to BYTES, so ``== "r"`` is + always False and every table/index name collision was reported as "no collision". +* ``test_blocking_sessions_sql_uses_to_regclass`` — ``'agent_messages'::regclass`` raises + UndefinedTable on a database that does not have the table yet, which crashed preflight + instead of reporting. +* ``test_harness_findings_flags_sql_before_begin_transaction`` — the env.py form that made + ``alembic upgrade`` log a full successful chain, exit 0 and commit nothing. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +_MIGRATE_DIR = Path(__file__).resolve().parents[2] / "scripts" / "migrate" + + +def _load(module_name: str, filename: str): + spec = importlib.util.spec_from_file_location(module_name, _MIGRATE_DIR / filename) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +# postflight imports preflight itself, under the name "copi_migrate_preflight"; load it +# first so both this file and postflight share one instance. +pf = _load("copi_migrate_preflight", "preflight.py") +po = _load("copi_migrate_postflight", "postflight.py") + + +# --------------------------------------------------------------------------- # +# Exit-code contract +# --------------------------------------------------------------------------- # + + +def test_exit_code_constants_match_the_documented_contract(): + assert (pf.EXIT_OK, pf.EXIT_BLOCKED, pf.EXIT_WARN) == (0, 1, 2) + + +@pytest.mark.parametrize( + ("statuses", "expected"), + [ + ([], pf.EXIT_OK), + ([pf.PASS], pf.EXIT_OK), + ([pf.PASS, pf.PASS], pf.EXIT_OK), + ([pf.WARN], pf.EXIT_WARN), + ([pf.PASS, pf.WARN], pf.EXIT_WARN), + ([pf.BLOCK], pf.EXIT_BLOCKED), + ([pf.PASS, pf.WARN, pf.BLOCK], pf.EXIT_BLOCKED), + # BLOCK dominates however many WARNs precede it. + ([pf.WARN] * 5 + [pf.BLOCK], pf.EXIT_BLOCKED), + ], +) +def test_exit_code_for(statuses, expected): + assert pf.exit_code_for(statuses) == expected + + +def test_worst_status(): + assert pf.worst_status([]) == pf.PASS + assert pf.worst_status([pf.PASS, pf.WARN]) == pf.WARN + assert pf.worst_status([pf.WARN, pf.BLOCK]) == pf.BLOCK + + +def test_preflight_report_maps_warn_to_exit_2(): + r = pf.Report("preflight") + r.add("a", pf.PASS) + assert r.exit_code() == 0 + r.add("b", pf.WARN) + assert r.exit_code() == 2 + r.add("c", pf.BLOCK) + assert r.exit_code() == 1 + + +def test_postflight_report_maps_warn_to_exit_0(): + """postflight's contract is only 0 = verified / 1 = failed, so a WARN cannot exit 2.""" + r = pf.Report("postflight", warn_exit_code=0) + r.add("a", pf.WARN) + assert r.exit_code() == 0 + assert "VERIFIED WITH WARNINGS" in r.render_text() + r.add("b", pf.BLOCK) + assert r.exit_code() == 1 + assert "VERIFICATION FAILED" in r.render_text() + + +def test_report_renders_numbered_checklist_with_status_and_remediation(): + r = pf.Report("preflight") + r.add("first thing", pf.PASS, "all good") + r.add("second thing", pf.BLOCK, "broken", ["DROP INDEX foo;"]) + text = r.render_text() + assert " 1. [PASS ] first thing" in text + assert " 2. [BLOCK] second thing" in text + assert "DROP INDEX foo;" in text + assert "BLOCKED" in text + assert "exit 1" in text + + +def test_report_to_dict_verdict_reflects_statuses_not_the_remapped_exit_code(): + r = pf.Report("postflight", warn_exit_code=0) + r.add("a", pf.WARN) + d = r.to_dict() + assert d["exit_code"] == 0 + assert d["verdict"] == "warn" + assert [c["number"] for c in d["checks"]] == [1] + + +async def test_add_guarded_turns_an_exception_into_a_block_rather_than_a_traceback(): + r = pf.Report("preflight") + + def explode(): + raise RuntimeError("catalog on fire") + + await r.add_guarded("some check", explode) + assert r.statuses == [pf.BLOCK] + assert "catalog on fire" in r.checks[0].detail + assert r.exit_code() == 1 + + +# --------------------------------------------------------------------------- # +# Sizing: calibration and thresholds +# --------------------------------------------------------------------------- # + + +def test_lock_window_constants_reproduce_the_measurements_they_were_fitted_to(): + """Measured DDL-block totals: 112 ms @10k, 747 ms @100k, 7,902 ms @1M rows. + + The model is a floor for an idle server with a warm cache, so it is allowed to be + optimistic — but not by more than 25% at any calibration point, or the number quoted + to the operator stops meaning anything. + """ + for rows, measured_ms in ((10_011, 112.0), (100_011, 747.2), (1_000_011, 7901.8)): + floor_ms, _ = pf.estimate_lock_window_ms(rows) + assert abs(floor_ms - measured_ms) / measured_ms < 0.25, (rows, floor_ms, measured_ms) + + +def test_lock_window_ceiling_is_the_contention_factor_times_the_floor(): + floor_ms, ceiling_ms = pf.estimate_lock_window_ms(500_000) + assert ceiling_ms == pytest.approx(floor_ms * pf.LOCK_WINDOW_CONTENTION_FACTOR) + + +def test_lock_window_is_monotone_and_defined_at_zero_and_for_nonsense_input(): + assert pf.estimate_lock_window_ms(0)[0] == pf.LOCK_WINDOW_FIXED_MS + # A negative count can only come from a broken caller; it must not produce a + # smaller-than-fixed (or negative) estimate that would read as "instant". + assert pf.estimate_lock_window_ms(-5)[0] == pf.LOCK_WINDOW_FIXED_MS + prev = -1.0 + for rows in (0, 1_000, 100_000, 1_000_000, 10_000_000): + floor_ms = pf.estimate_lock_window_ms(rows)[0] + assert floor_ms > prev + prev = floor_ms + + +def test_sizing_status_passes_for_a_short_window(): + rows = 100_000 + _, ceiling = pf.estimate_lock_window_ms(rows) + status, note = pf.sizing_status(rows, ceiling) + assert status == pf.PASS + assert "lock window" in note + + +def test_sizing_status_warns_once_the_window_needs_scheduling(): + status, note = pf.sizing_status(900_000, pf.LOCK_WINDOW_WARN_MS + 1) + assert status == pf.WARN + assert "schedule a window" in note + + +def test_sizing_status_warns_that_it_is_extrapolating_beyond_the_calibrated_range(): + rows = pf.LOCK_WINDOW_CALIBRATED_MAX_ROWS + 1 + status, note = pf.sizing_status(rows, 1.0) + assert status == pf.WARN + assert "extrapolation" in note + + +def test_index_growth_fraction_matches_the_measured_growth(): + """Measured post/pre total relation size: 3608/2016, 34/19 MB, 339/188 MB -> ~+80%.""" + for pre, post in ((2016.0, 3608.0), (19.0, 34.0), (188.0, 339.0)): + measured = (post - pre) / pre + assert abs(pf.INDEX_GROWTH_FRACTION - measured) < 0.05, (pre, post, measured) + + +# --------------------------------------------------------------------------- # +# Revision classification +# --------------------------------------------------------------------------- # + + +def test_revision_status_blocks_when_the_database_has_never_been_stamped(): + status, reason = pf.revision_status(None, "0023") + assert status == pf.BLOCK + assert "never been stamped" in reason + + +def test_revision_status_passes_at_the_target(): + status, reason = pf.revision_status("0023", "0023") + assert status == pf.PASS + assert "no-op" in reason + + +@pytest.mark.parametrize("rev", ["0018", "0019", "0020", "0021"]) +def test_revision_status_passes_at_a_supported_starting_point(rev): + assert pf.revision_status(rev, "0023")[0] == pf.PASS + + +@pytest.mark.parametrize("rev", ["0001", "0017", "0022", "0024", "abcdef"]) +def test_revision_status_blocks_anywhere_else(rev): + status, reason = pf.revision_status(rev, "0023") + assert status == pf.BLOCK + assert rev in reason + + +def test_supported_start_revisions_are_exactly_the_documented_set(): + assert pf.SUPPORTED_START_REVISIONS == ("0018", "0019", "0020", "0021", "0023") + assert pf.DEFAULT_TARGET == "0024" + + +def test_0021_is_supported_because_that_is_origin_mains_own_alembic_head(): + """Regression guard: do not narrow this list back to ("0018", "0019"). + + origin/main's head is 0021 (PR19 merged 0019, 0020 and 0021), so any deployment + tracking main is stamped 0021. The first version of this allowlist blocked exactly + that state -- preflight refused the one starting point main itself produces, which + was found by auditing the branch for a PR into main rather than by any test. + + 0022 is deliberately NOT here: no deployment reaches it (main stops at 0021, this + branch's head is 0023) and the path has not been exercised from there. An allowlist + for a safety gate should contain what was tested, not what seems plausible. + """ + assert "0021" in pf.SUPPORTED_START_REVISIONS + assert "0020" in pf.SUPPORTED_START_REVISIONS + assert "0022" not in pf.SUPPORTED_START_REVISIONS + + +def test_sizing_does_not_quote_the_0019_index_build_once_0019_has_run(): + """The row-scaled lock estimate only applies while 0019 is still pending.""" + assert pf.POST_0019_STARTS == ("0020", "0021") + for rev in pf.POST_0019_STARTS: + assert rev in pf.SUPPORTED_START_REVISIONS + + +# --------------------------------------------------------------------------- # +# lock_timeout resolution +# --------------------------------------------------------------------------- # + + +def test_lock_timeout_prefers_the_environment_variable(): + value, source = pf.resolve_lock_timeout_ms({"ALEMBIC_LOCK_TIMEOUT_MS": "250"}, "irrelevant") + assert value == "250" + assert "environment" in source + + +def test_lock_timeout_falls_back_to_the_env_py_default(): + src = 'LOCK_TIMEOUT_MS = os.environ.get("ALEMBIC_LOCK_TIMEOUT_MS", "10000")\n' + value, source = pf.resolve_lock_timeout_ms({}, src) + assert value == "10000" + assert "env.py" in source + + +def test_lock_timeout_reports_zero_when_env_py_sets_none(): + value, source = pf.resolve_lock_timeout_ms({}, "def do_run_migrations(c): pass\n") + assert value == "0" + assert "wait forever" in source + + +def test_lock_timeout_is_unknown_when_env_py_cannot_be_read(): + assert pf.resolve_lock_timeout_ms({}, None) == ("unknown", "could not determine") + + +# --------------------------------------------------------------------------- # +# The migration-harness analyser +# --------------------------------------------------------------------------- # + +_BROKEN_ENV_PY = ''' +LOCK_TIMEOUT_MS = os.environ.get("ALEMBIC_LOCK_TIMEOUT_MS", "10000") + + +def do_run_migrations(connection: Connection) -> None: + if LOCK_TIMEOUT_MS and LOCK_TIMEOUT_MS != "0": + connection.exec_driver_sql(f"SET lock_timeout = {int(LOCK_TIMEOUT_MS)}") + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() +''' + +_FIXED_ENV_PY = ''' +LOCK_TIMEOUT_MS = os.environ.get("ALEMBIC_LOCK_TIMEOUT_MS", "10000") + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() +''' + + +def test_harness_findings_flags_sql_before_begin_transaction(): + """The bug: `alembic upgrade` logs the whole chain, exits 0, and commits nothing. + + Reproduced against a throwaway database: with this form, `alembic upgrade 0018` + reported every revision applied and left the database with ZERO tables and no + alembic_version; with ALEMBIC_LOCK_TIMEOUT_MS=0 (which skips the statement) the same + command left 25 tables and version 0018. + """ + findings = pf.harness_findings(_BROKEN_ENV_PY) + assert len(findings) == 1 + assert "connection.exec_driver_sql" in findings[0] + assert "before context.begin_transaction()" in findings[0] + + +def test_harness_findings_clean_when_the_timeout_is_set_on_the_engine_instead(): + assert pf.harness_findings(_FIXED_ENV_PY) == [] + + +@pytest.mark.parametrize("method", ["execute", "exec_driver_sql", "scalar", "scalars", "begin"]) +def test_harness_findings_covers_every_autobegin_method(method): + src = f"def do_run_migrations(connection):\n connection.{method}('x')\n" + assert len(pf.harness_findings(src)) == 1 + + +def test_harness_findings_ignores_calls_on_something_other_than_the_connection(): + src = "def do_run_migrations(connection):\n other.execute('x')\n context.configure(1)\n" + assert pf.harness_findings(src) == [] + + +def test_harness_findings_ignores_other_functions(): + src = "def run_migrations_offline():\n connection.execute('x')\n" + assert pf.harness_findings(src) == [] + + +def test_harness_findings_survives_a_syntax_error(): + findings = pf.harness_findings("def broken(:\n") + assert len(findings) == 1 + assert "could not parse" in findings[0] + + +def test_the_real_env_py_in_this_tree_is_clean(): + """Regression guard. This file has carried the bug once; it must not again.""" + src = pf.read_env_py() + assert src is not None, "alembic/env.py should be readable from the repo root" + assert pf.harness_findings(src) == [] + + +# --------------------------------------------------------------------------- # +# Blocking-session classification +# --------------------------------------------------------------------------- # + + +def _session(state="active", xact_age_s=0.0, pid=1): + return { + "pid": pid, + "state": state, + "usename": "copi", + "application_name": "", + "xact_age_s": xact_age_s, + "query_age_s": xact_age_s, + "query": "SELECT 1", + "holds_agent_messages_lock": False, + } + + +def test_no_sessions_passes(): + assert pf.blocking_sessions_status([])[0] == pf.PASS + + +@pytest.mark.parametrize("age", [0.01, 1.0, 3600.0]) +def test_idle_in_transaction_blocks_at_any_age(age): + status, detail = pf.blocking_sessions_status([_session("idle in transaction", age)]) + assert status == pf.BLOCK + assert "queues ahead of new readers" in detail + + +def test_idle_in_transaction_aborted_also_blocks(): + status, _ = pf.blocking_sessions_status([_session("idle in transaction (aborted)", 2.0)]) + assert status == pf.BLOCK + + +def test_a_long_open_transaction_blocks(): + status, detail = pf.blocking_sessions_status( + [_session("active", pf.DEFAULT_MAX_TOLERABLE_XACT_AGE_S + 1)] + ) + assert status == pf.BLOCK + assert "open longer than" in detail + + +def test_a_short_open_transaction_only_warns(): + """A sub-threshold query releases its lock on its own; blocking on it is crying wolf.""" + status, _ = pf.blocking_sessions_status( + [_session("active", pf.DEFAULT_MAX_TOLERABLE_XACT_AGE_S - 1)] + ) + assert status == pf.WARN + + +def test_an_active_session_with_no_transaction_only_warns(): + status, detail = pf.blocking_sessions_status([_session("active", 0.0)]) + assert status == pf.WARN + assert "blocked (not blocking)" in detail + + +def test_the_threshold_is_configurable(): + session = [_session("active", 4.0)] + assert pf.blocking_sessions_status(session, max_xact_age_s=10.0)[0] == pf.WARN + assert pf.blocking_sessions_status(session, max_xact_age_s=1.0)[0] == pf.BLOCK + + +# --------------------------------------------------------------------------- # +# Backup verdict table +# --------------------------------------------------------------------------- # + + +def _good_backup(**over): + facts = pf.BackupFacts( + path="/tmp/copi.sql.gz", + exists=True, + size_bytes=8_000, + age_hours=1.0, + fmt="gzip", + scanned=True, + has_agent_messages_ddl=True, + has_agent_messages_data=True, + live_agent_messages_rows=1_000, + ) + for k, v in over.items(): + setattr(facts, k, v) + return facts + + +def test_a_recent_data_bearing_dump_passes(): + status, notes = pf.evaluate_backup(_good_backup()) + assert status == pf.PASS + assert "data section present" in notes[0] + + +def test_no_backup_blocks_and_says_why_rollback_needs_one(): + status, notes = pf.evaluate_backup(pf.BackupFacts()) + assert status == pf.BLOCK + assert "no backup found" in notes[0] + assert any("pg_dump" in n for n in notes) + + +def test_a_stale_backup_blocks(): + status, notes = pf.evaluate_backup(_good_backup(age_hours=48.0), max_age_hours=24.0) + assert status == pf.BLOCK + assert "48.0h old" in notes[0] + + +def test_a_backup_exactly_at_the_age_threshold_is_accepted(): + assert pf.evaluate_backup(_good_backup(age_hours=24.0), max_age_hours=24.0)[0] == pf.PASS + + +def test_a_truncated_backup_blocks_on_size(): + status, notes = pf.evaluate_backup(_good_backup(size_bytes=200), min_bytes=1024) + assert status == pf.BLOCK + assert "under the" in notes[0] + + +def test_an_unreadable_backup_blocks(): + """A truncated gzip raises EOFError, not OSError; inspect_backup records it here.""" + status, notes = pf.evaluate_backup( + _good_backup(read_error="EOFError: Compressed file ended before the end-of-stream marker") + ) + assert status == pf.BLOCK + assert "EOFError" in notes[0] + + +def test_an_unrecognisable_file_blocks(): + status, notes = pf.evaluate_backup(_good_backup(fmt="unknown")) + assert status == pf.BLOCK + assert "not a recognisable pg_dump output" in notes[0] + + +def test_a_dump_of_some_other_database_blocks(): + status, notes = pf.evaluate_backup(_good_backup(has_agent_messages_ddl=False)) + assert status == pf.BLOCK + assert "not a dump of this database" in notes[0] + + +def test_a_schema_only_dump_blocks_when_the_live_table_has_rows(): + status, notes = pf.evaluate_backup( + _good_backup(has_agent_messages_data=False, live_agent_messages_rows=42) + ) + assert status == pf.BLOCK + assert "--schema-only" in notes[0] + + +def test_a_dump_with_no_data_section_is_fine_when_the_table_is_genuinely_empty(): + """Not crying wolf: an empty agent_messages produces no COPY section, correctly.""" + status, _ = pf.evaluate_backup( + _good_backup(has_agent_messages_data=False, live_agent_messages_rows=0) + ) + assert status == pf.PASS + + +def test_a_custom_format_archive_warns_because_pg_restore_is_not_installed(): + status, notes = pf.evaluate_backup(_good_backup(fmt="custom")) + assert status == pf.WARN + assert "pg_restore" in notes[0] + + +def test_the_override_warns_loudly_and_never_passes(): + status, notes = pf.evaluate_backup(pf.BackupFacts(override_reason="EBS snapshot 20:00Z")) + assert status == pf.WARN + assert "EBS snapshot 20:00Z" in notes[0] + assert "destroys agent_messages.content" in notes[0] + + +def test_backup_thresholds_are_the_documented_defaults(): + assert pf.DEFAULT_BACKUP_MAX_AGE_HOURS == 24.0 + assert pf.DEFAULT_BACKUP_MIN_BYTES == 1024 + + +# --------------------------------------------------------------------------- # +# Dump scanning and discovery (filesystem only, no database) +# --------------------------------------------------------------------------- # + +_PLAIN_DUMP_WITH_DATA = """\ +-- +-- PostgreSQL database dump +-- +CREATE TABLE public.agent_messages (id uuid NOT NULL); +COPY public.agent_messages (id, content) FROM stdin; +1\thello +\\. +""" + +_PLAIN_DUMP_SCHEMA_ONLY = """\ +-- +-- PostgreSQL database dump +-- +CREATE TABLE public.agent_messages (id uuid NOT NULL); +CREATE INDEX ix_agent_messages_run_posted ON public.agent_messages USING btree (id); +""" + + +def test_scan_dump_text_finds_a_copy_data_section(): + assert pf.scan_dump_text(_PLAIN_DUMP_WITH_DATA) == (True, True) + + +def test_scan_dump_text_distinguishes_a_schema_only_dump(): + assert pf.scan_dump_text(_PLAIN_DUMP_SCHEMA_ONLY) == (True, False) + + +def test_scan_dump_text_finds_an_insert_style_data_section(): + body = "CREATE TABLE public.agent_messages (id uuid);\nINSERT INTO public.agent_messages VALUES (1);\n" + assert pf.scan_dump_text(body) == (True, True) + + +def test_scan_dump_text_reports_nothing_for_an_unrelated_dump(): + assert pf.scan_dump_text("CREATE TABLE public.users (id uuid);\n") == (False, False) + + +def test_inspect_backup_records_a_truncated_gzip_instead_of_raising(tmp_path): + """This crashed the whole script before the fix; a bad backup must be a finding.""" + import gzip + + good = tmp_path / "d.sql.gz" + good.write_bytes(gzip.compress(_PLAIN_DUMP_WITH_DATA.encode())) + truncated = tmp_path / "t.sql.gz" + truncated.write_bytes(good.read_bytes()[:20]) + + facts = pf.inspect_backup(truncated, live_rows=5, override=None) + assert facts.exists + assert facts.fmt == "gzip" + assert facts.read_error is not None + assert pf.evaluate_backup(facts)[0] == pf.BLOCK + + +def test_inspect_backup_reads_a_real_gzipped_plain_dump(tmp_path): + import gzip + + p = tmp_path / "d.sql.gz" + p.write_bytes(gzip.compress(_PLAIN_DUMP_WITH_DATA.encode())) + facts = pf.inspect_backup(p, live_rows=5, override=None) + assert (facts.fmt, facts.has_agent_messages_ddl, facts.has_agent_messages_data) == ( + "gzip", + True, + True, + ) + # min_bytes=1: this hand-written dump gzips to a couple of hundred bytes, well under + # the production floor, and the floor is not what is under test here. + assert pf.evaluate_backup(facts, min_bytes=1)[0] == pf.PASS + + +def test_inspect_backup_sniffs_the_custom_format_magic(tmp_path): + p = tmp_path / "d.dump" + p.write_bytes(b"PGDMP" + b"\x00" * 4096) + assert pf.inspect_backup(p, live_rows=5, override=None).fmt == "custom" + + +def test_inspect_backup_marks_a_non_dump_as_unknown(tmp_path): + p = tmp_path / "notes.txt" + p.write_text("just some notes\n" * 100) + assert pf.inspect_backup(p, live_rows=5, override=None).fmt == "unknown" + + +def test_inspect_backup_on_a_missing_path_reports_not_exists(tmp_path): + facts = pf.inspect_backup(tmp_path / "nope.sql.gz", live_rows=5, override=None) + assert not facts.exists + + +def test_find_backup_takes_the_newest_file_in_a_directory(tmp_path): + import os + import time + + old = tmp_path / "old.sql.gz" + new = tmp_path / "new.sql.gz" + old.write_bytes(b"x" * 10) + new.write_bytes(b"y" * 10) + now = time.time() + os.utime(old, (now - 7200, now - 7200)) + os.utime(new, (now - 60, now - 60)) + assert pf.find_backup(str(tmp_path)) == new + + +def test_find_backup_ignores_files_that_are_not_dumps(tmp_path): + (tmp_path / "readme.md").write_text("hi") + assert pf.find_backup(str(tmp_path)) is None + + +def test_find_backup_accepts_a_file_path_directly(tmp_path): + p = tmp_path / "explicit.sql" + p.write_text("x") + assert pf.find_backup(str(p)) == p + + +def test_find_backup_returns_none_for_a_path_that_does_not_exist(tmp_path): + assert pf.find_backup(str(tmp_path / "missing.sql.gz")) is None + + +# --------------------------------------------------------------------------- # +# Legacy-row inventory +# --------------------------------------------------------------------------- # + + +def test_legacy_inventory_passes_when_nothing_will_be_empty(): + assert pf.legacy_inventory_status(0, 0)[0] == pf.PASS + + +def test_legacy_inventory_names_slack_recoverable_rows(): + status, note = pf.legacy_inventory_status(12, 0) + assert status == pf.WARN + assert "12 Slack-recoverable" in note + assert "UNRECOVERABLE" not in note + + +def test_legacy_inventory_shouts_about_permanently_unrecoverable_rows(): + status, note = pf.legacy_inventory_status(0, 7) + assert status == pf.WARN + assert "7 PERMANENTLY UNRECOVERABLE" in note + + +def test_legacy_inventory_reports_both_buckets_separately(): + _, note = pf.legacy_inventory_status(3, 4) + assert "3 Slack-recoverable" in note + assert "4 PERMANENTLY UNRECOVERABLE" in note + + +# --------------------------------------------------------------------------- # +# Row-count comparison (the preflight -> postflight handoff) +# --------------------------------------------------------------------------- # + + +def test_identical_counts_compare_clean(): + counts = {"users": 3, "agent_messages": 18} + ok, problems = pf.compare_row_counts(counts, dict(counts)) + assert ok and problems == [] + + +def test_row_loss_always_fails(): + ok, problems = pf.compare_row_counts({"agent_messages": 18}, {"agent_messages": 17}) + assert not ok + assert "1 rows LOST" in problems[0] + + +def test_row_loss_fails_even_with_allow_row_growth(): + ok, _ = pf.compare_row_counts( + {"agent_messages": 18}, {"agent_messages": 17}, allow_growth=True + ) + assert not ok + + +def test_row_growth_fails_by_default_because_the_migration_inserts_nothing(): + ok, problems = pf.compare_row_counts({"agent_messages": 18}, {"agent_messages": 19}) + assert not ok + assert "a writer was live" in problems[0] + + +def test_row_growth_can_be_allowed_explicitly(): + ok, problems = pf.compare_row_counts( + {"agent_messages": 18}, {"agent_messages": 19}, allow_growth=True + ) + assert ok and problems == [] + + +def test_a_table_that_disappeared_fails(): + ok, problems = pf.compare_row_counts({"users": 3}, {}) + assert not ok + assert "now MISSING" in problems[0] + + +def test_tables_the_chain_creates_are_not_flagged_as_unexpected(): + """0020 creates pi_dm_messages and 0022 creates three cohort tables, so they are + absent from the preflight snapshot by construction. Flagging them would be a false + failure on every single successful migration.""" + ok, problems = pf.compare_row_counts( + {"users": 3}, + {"users": 3, "pi_dm_messages": 0, "cohorts": 0}, + expected_new=po.CHAIN_CREATED_TABLES, + ) + assert ok and problems == [] + + +def test_a_table_the_chain_does_not_create_is_still_flagged(): + ok, problems = pf.compare_row_counts( + {"users": 3}, {"users": 3, "mystery": 9}, expected_new=po.CHAIN_CREATED_TABLES + ) + assert not ok + assert "did not exist before" in problems[0] + + +def test_chain_created_tables_is_derived_from_planned_objects_not_relisted(): + assert po.CHAIN_CREATED_TABLES == frozenset( + o.name for o in pf.PLANNED_OBJECTS if o.kind == "table" + ) + assert po.CHAIN_CREATED_TABLES == { + "pi_dm_messages", + "cohorts", + "cohort_memberships", + "cohort_audit_events", + } + + +# --------------------------------------------------------------------------- # +# URL handling +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize( + "raw", + [ + "postgresql://copi:copi@postgres:5432/copi", + "postgres://copi:copi@postgres:5432/copi", + "postgresql+psycopg2://copi:copi@postgres:5432/copi", + "postgresql+psycopg://copi:copi@postgres:5432/copi", + "postgresql+asyncpg://copi:copi@postgres:5432/copi", + ], +) +def test_normalize_async_url_forces_the_asyncpg_driver(raw): + out = pf.normalize_async_url(raw) + assert out == "postgresql+asyncpg://copi:copi@postgres:5432/copi" + + +def test_normalize_async_url_leaves_an_unrecognised_scheme_alone(): + assert pf.normalize_async_url("sqlite+aiosqlite:///x.db") == "sqlite+aiosqlite:///x.db" + + +def test_redact_url_hides_the_password_but_keeps_the_rest(): + out = pf.redact_url("postgresql+asyncpg://copi:s3cret@postgres:5432/copi") + assert out == "postgresql+asyncpg://copi:***@postgres:5432/copi" + assert "s3cret" not in out + + +def test_redact_url_is_a_no_op_when_there_is_no_password(): + assert pf.redact_url("postgresql+asyncpg://postgres:5432/copi").count("***") == 0 + + +# --------------------------------------------------------------------------- # +# Planned objects: the collision check's input +# --------------------------------------------------------------------------- # + + +def test_planned_objects_between_0018_and_the_target_is_everything(): + assert set(pf.planned_objects_between("0018", pf.DEFAULT_TARGET)) == set(pf.PLANNED_OBJECTS) + + +def test_planned_objects_between_0019_and_0023_excludes_what_0019_already_made(): + planned = pf.planned_objects_between("0019", "0023") + names = {o.name for o in planned} + # Already present at 0019, so their existence is correct rather than a collision. + assert "content" not in names + assert "uq_agent_messages_run_ts" not in names + assert "ix_agent_messages_run_posted" not in names + # Still to come. + assert "pi_dm_messages" in names + assert "cohorts" in names + assert "ix_agent_messages_run_created" in names + + +def test_planned_objects_between_a_revision_and_itself_is_empty(): + assert pf.planned_objects_between("0023", "0023") == () + + +def test_planned_objects_kinds_are_all_understood_by_the_collision_check(): + assert {o.kind for o in pf.PLANNED_OBJECTS} <= { + "table", + "column", + "index", + "constraint", + "type", + } + + +def test_every_planned_column_names_its_table(): + for obj in pf.PLANNED_OBJECTS: + if obj.kind in {"column", "index", "constraint"}: + assert obj.table, obj + + +def test_planned_objects_matches_what_the_migration_files_actually_create(): + """Drift guard: re-derive the object names from alembic/versions/0019..0023 and + compare. Hardcoding the list keeps the check readable; this keeps it honest.""" + import re + + versions_dir = Path(__file__).resolve().parents[2] / "alembic" / "versions" + patterns = { + "index": re.compile(r'create_index\(\s*\n?\s*"([^"]+)"'), + "table": re.compile(r'create_table\(\s*\n?\s*"([^"]+)"'), + "column": re.compile(r'add_column\(\s*\n?\s*"[^"]+",\s*\n?\s*sa\.Column\("([^"]+)"'), + "constraint": re.compile(r'create_unique_constraint\(\s*\n?\s*"([^"]+)"'), + } + for revision in ("0019", "0020", "0021", "0022", "0023", "0024"): + matches = list(versions_dir.glob(f"{revision}_*.py")) + assert len(matches) == 1, (revision, matches) + source = matches[0].read_text() + upgrade = source.split("def upgrade()", 1)[1].split("def downgrade()", 1)[0] + declared = { + o.name for o in pf.PLANNED_OBJECTS if o.revision == revision + } + for kind, pattern in patterns.items(): + found = set(pattern.findall(upgrade)) + missing = found - declared + assert not missing, ( + f"{revision} creates {kind}(s) {sorted(missing)} that PLANNED_OBJECTS " + "does not list — the collision check would miss them" + ) + # Inline UniqueConstraint(...) inside create_table, plus inline sa.Enum types. + for name in re.findall(r'sa\.UniqueConstraint\([^)]*name="([^"]+)"', upgrade): + assert name in declared, (revision, name) + for name in re.findall(r'name="([a-z_]+_enum)"', upgrade): + assert name in declared, (revision, name) + + +# --------------------------------------------------------------------------- # +# The alembic script-directory guard (mirrors scripts/ci.sh, but relates it to the +# stamped revision). No database: check_alembic_scripts only reads files. +# --------------------------------------------------------------------------- # + + +def _fake_versions(tmp_path, revisions): + """revisions: list of (revision_id, down_revision or None, filename).""" + versions = tmp_path / "alembic" / "versions" + versions.mkdir(parents=True) + for rev, down, filename in revisions: + down_literal = f'"{down}"' if down else "None" + (versions / filename).write_text( + f'revision: str = "{rev}"\ndown_revision: Union[str, None] = {down_literal}\n' + ) + return tmp_path + + +LINEAR_TREE = [ + ("0018", "0017", "0018_a.py"), + ("0019", "0018", "0019_b.py"), + ("0020", "0019", "0020_c.py"), + ("0017", None, "0017_root.py"), +] + + +async def test_alembic_scripts_passes_on_a_single_head_matching_the_target(monkeypatch, tmp_path): + monkeypatch.setattr(pf, "REPO_ROOT", _fake_versions(tmp_path, LINEAR_TREE)) + title, status, detail, _rem, data = await pf.check_alembic_scripts("0018", "0020") + assert status == pf.PASS + assert data["heads"] == ["0020"] + assert "4 migration files" in detail + + +async def test_alembic_scripts_blocks_on_duplicate_revision_ids(monkeypatch, tmp_path): + """The historical case: 0019_agent_message_content.py and 0019_add_cohorts.py both + declared revision = "0019". A targeted upgrade applies whichever sorts last and + stamps the database as fully migrated.""" + tree = [*LINEAR_TREE, ("0019", "0018", "0019_add_cohorts.py")] + monkeypatch.setattr(pf, "REPO_ROOT", _fake_versions(tmp_path, tree)) + _title, status, detail, rem, data = await pf.check_alembic_scripts("0018", "0020") + assert status == pf.BLOCK + assert "DUPLICATE revision ids" in detail + assert "0019" in data["duplicates"] + assert sorted(data["duplicates"]["0019"]) == ["0019_add_cohorts.py", "0019_b.py"] + assert any("uniq -d" in r for r in rem) + + +async def test_alembic_scripts_blocks_on_two_heads(monkeypatch, tmp_path): + tree = [*LINEAR_TREE, ("0021", "0019", "0021_branch.py")] + monkeypatch.setattr(pf, "REPO_ROOT", _fake_versions(tmp_path, tree)) + _title, status, detail, _rem, data = await pf.check_alembic_scripts("0018", "0020") + assert status == pf.BLOCK + assert "Expected exactly one head" in detail + assert sorted(data["heads"]) == ["0020", "0021"] + + +async def test_alembic_scripts_warns_when_the_head_is_not_the_requested_target( + monkeypatch, tmp_path +): + monkeypatch.setattr(pf, "REPO_ROOT", _fake_versions(tmp_path, LINEAR_TREE)) + _title, status, detail, rem, _data = await pf.check_alembic_scripts("0018", "0019") + assert status == pf.WARN + assert "not the requested target" in detail + assert any("--target 0020" in r for r in rem) + + +async def test_alembic_scripts_blocks_when_the_stamp_exists_in_no_migration_file( + monkeypatch, tmp_path +): + """A database migrated by a different branch: nothing here can be trusted about it.""" + monkeypatch.setattr(pf, "REPO_ROOT", _fake_versions(tmp_path, LINEAR_TREE)) + _title, status, detail, _rem, _data = await pf.check_alembic_scripts("0099", "0020") + assert status == pf.BLOCK + assert "which no migration file defines" in detail + + +async def test_alembic_scripts_tolerates_an_unstamped_database(monkeypatch, tmp_path): + monkeypatch.setattr(pf, "REPO_ROOT", _fake_versions(tmp_path, LINEAR_TREE)) + _title, status, _detail, _rem, _data = await pf.check_alembic_scripts(None, "0020") + assert status == pf.PASS + + +async def test_alembic_scripts_agrees_with_the_real_tree(): + """The live repo must have exactly one head and no duplicate ids — the same property + scripts/ci.sh gates on, asserted here against the same files.""" + _title, status, _detail, _rem, data = await pf.check_alembic_scripts(None, pf.DEFAULT_TARGET) + assert status == pf.PASS + assert data["heads"] == [pf.DEFAULT_TARGET] + + +def test_revision_order_covers_the_supported_range(): + assert pf.REVISION_ORDER[0] == "0018" + assert pf.REVISION_ORDER[-1] == pf.DEFAULT_TARGET + assert list(pf.REVISION_ORDER) == sorted(pf.REVISION_ORDER) + + +# --------------------------------------------------------------------------- # +# Query builders +# --------------------------------------------------------------------------- # + + +def test_duplicate_groups_sql_finds_every_group_in_one_pass(): + sql = " ".join(pf.DUPLICATE_GROUPS_SQL.split()) + # All groups, not the first one: GROUP BY + HAVING, with no LIMIT anywhere. + assert "GROUP BY simulation_run_id, message_ts" in sql + assert "HAVING count(*) > 1" in sql + assert "LIMIT" not in sql.upper() + # The row ids, so the operator can act without a second query. + assert "array_agg(id::text ORDER BY created_at, id)" in sql + # NULL message_ts is exempt from a Postgres UNIQUE constraint, so including those + # rows would report duplicates that cannot fail the migration. + assert "WHERE message_ts IS NOT NULL" in sql + + +def test_the_remediation_sql_partitions_on_the_constraint_columns(): + for sql in (pf.DEDUPE_DELETE_SQL, pf.DEDUPE_NULL_SQL): + assert "PARTITION BY simulation_run_id, message_ts" in sql + assert "ORDER BY created_at, id" in sql + assert "rn > 1" in sql + # Wrapped in a transaction so a partial remediation cannot be left behind. + assert sql.strip().startswith("--") + assert "BEGIN;" in sql and "COMMIT;" in sql + + +def test_the_two_remediations_differ_in_exactly_the_destructive_step(): + assert "DELETE FROM agent_messages" in pf.DEDUPE_DELETE_SQL + assert "DELETE FROM agent_messages" not in pf.DEDUPE_NULL_SQL + assert "SET message_ts = NULL" in pf.DEDUPE_NULL_SQL + + +def test_blocking_sessions_sql_uses_to_regclass(): + """Regression guard. `'agent_messages'::regclass` raises UndefinedTable on a database + that does not have the table yet, which crashed preflight instead of reporting.""" + sql = pf.BLOCKING_SESSIONS_SQL + assert "to_regclass('public.agent_messages')" in sql + # Comment lines mention the broken form on purpose; strip them before asserting. + executable = "\n".join( + line for line in sql.splitlines() if not line.strip().startswith("--") + ) + assert "::regclass" not in executable + + +def test_blocking_sessions_sql_excludes_our_own_sessions(): + sql = pf.BLOCKING_SESSIONS_SQL + assert "a.pid <> pg_backend_pid()" in sql + assert ":app_name" in sql + assert pf.APPLICATION_NAME + + +def test_existing_object_names_casts_relkind_to_text(): + """Regression guard for a check that failed OPEN. + + pg_class.relkind is Postgres' internal "char" type and asyncpg decodes it to BYTES, + so `row["k"] == "r"` was always False and the table/index sets were always empty — + every collision was reported as "none exists yet". Verified against a fixture with a + pre-existing ix_agent_messages_run_posted, which really does abort migration 0019. + """ + import inspect + + source = inspect.getsource(pf.existing_object_names) + assert "c.relkind::text" in source + assert "c.relkind AS k" not in source + + +def test_snapshot_row_counts_uses_exact_counts_not_reltuples(): + import inspect + + source = inspect.getsource(pf.snapshot_row_counts) + assert "count(*)" in source + assert "reltuples" not in source.split('"""')[2] + + +# --------------------------------------------------------------------------- # +# postflight's expectations +# --------------------------------------------------------------------------- # + + +def test_postflight_expects_agent_id_to_have_become_nullable(): + """0019 RELAXES agent_messages.agent_id. If postflight expected NOT NULL it would + pass on a database where 0019 never ran.""" + spec = [c for c in po.EXPECTED_COLUMNS if c[:2] == ("agent_messages", "agent_id")] + assert len(spec) == 1 + assert spec[0][3] is True + + +def test_postflight_expects_the_0023_columns_to_stay_nullable_and_unbackfilled(): + for column in ("synthesis_validated", "evidence_pmid_count", "evidence_pub_count"): + spec = [c for c in po.EXPECTED_COLUMNS if c[:2] == ("researcher_profiles", column)] + assert len(spec) == 1, column + assert spec[0][3] is True, column + + +def test_postflight_pins_the_content_columns_as_not_null_with_their_server_defaults(): + expected = { + "content": "''::text", + "sender_name": "''::character varying", + "is_bot": "true", + "posted_at": "'0'::double precision", + } + for column, default in expected.items(): + spec = [c for c in po.EXPECTED_COLUMNS if c[:2] == ("agent_messages", column)] + assert len(spec) == 1, column + assert spec[0][3] is False, column + assert spec[0][4] == default, column + + +def test_postflight_keeps_the_partial_predicate_in_the_expected_index_definition(): + """Without the predicate the index is a different, much larger object.""" + assert "WHERE (slack_ts IS NOT NULL)" in po.EXPECTED_INDEXES["ix_agent_messages_run_slack_ts"] + + +def test_postflight_expects_an_index_for_every_index_the_chain_creates(): + planned = {o.name for o in pf.PLANNED_OBJECTS if o.kind in {"index", "constraint"}} + assert planned <= set(po.EXPECTED_INDEXES) + + +def test_postflight_expects_a_table_for_every_table_the_chain_creates(): + planned = {o.name for o in pf.PLANNED_OBJECTS if o.kind == "table"} + assert planned == set(po.EXPECTED_TABLES) + + +def test_postflight_expects_a_column_for_every_column_the_chain_creates(): + planned = {(o.table, o.name) for o in pf.PLANNED_OBJECTS if o.kind == "column"} + expected = {(t, c) for (t, c, _dt, _n, _d) in po.EXPECTED_COLUMNS} + assert planned <= expected + + +def test_postflight_expects_the_enum_the_chain_creates(): + planned = {o.name for o in pf.PLANNED_OBJECTS if o.kind == "type"} + assert planned == set(po.EXPECTED_ENUMS) + assert po.EXPECTED_ENUMS["pi_dm_direction_enum"] == ("inbound", "outbound") + + +def test_must_be_non_null_is_derived_from_expected_columns(): + assert set(po.MUST_BE_NON_NULL) == { + (t, c) for (t, c, _dt, nullable, _d) in po.EXPECTED_COLUMNS if not nullable + } + assert ("agent_messages", "content") in po.MUST_BE_NON_NULL + assert ("agent_messages", "agent_id") not in po.MUST_BE_NON_NULL + + +def test_drift_classification_fails_on_what_a_dropped_object_produces(): + """Sabotage-verified: dropping a column yields add_column, dropping an index yields + add_index, relaxing a NOT NULL yields modify_nullable.""" + for op in ("add_column", "add_index", "add_table", "modify_nullable"): + assert op in po.DRIFT_FAIL_OPS + + +def test_drift_classification_ignores_only_the_pre_existing_noise(): + """25 differences are reported on a correctly migrated database; all are the DB + having something the ORM never declared, which is harmless.""" + for op in ("remove_index", "remove_constraint", "add_table_comment"): + assert op in po.DRIFT_IGNORED_OPS + assert not (po.DRIFT_FAIL_OPS & po.DRIFT_IGNORED_OPS) + + +def test_postflight_status_aliases_are_the_same_tokens_preflight_uses(): + assert (po.PASS, po.WARN, po.FAIL) == (pf.PASS, pf.WARN, pf.BLOCK) + + +# --------------------------------------------------------------------------- # +# CLI surface +# --------------------------------------------------------------------------- # + + +def test_preflight_parser_defaults(): + args = pf.build_parser().parse_args([]) + assert args.database_url is None + assert args.target == "0024" + assert args.json is False + assert args.snapshot is None + assert args.backup_path is None + assert args.backup_max_age_hours == pf.DEFAULT_BACKUP_MAX_AGE_HOURS + assert args.backup_min_bytes == pf.DEFAULT_BACKUP_MIN_BYTES + assert args.backup_verified_elsewhere is None + assert args.max_xact_age_s == pf.DEFAULT_MAX_TOLERABLE_XACT_AGE_S + assert args.statement_timeout_ms == 60_000 + + +def test_preflight_parser_accepts_the_documented_interface(): + args = pf.build_parser().parse_args( + [ + "--database-url", + "postgresql://u:p@h:5432/d", + "--target", + "0022", + "--json", + "--snapshot", + "/tmp/s.json", + "--backup-path", + "/tmp/b.sql.gz", + "--backup-max-age-hours", + "6", + "--max-duplicate-groups", + "5", + ] + ) + assert args.database_url == "postgresql://u:p@h:5432/d" + assert args.target == "0022" + assert args.json is True + assert args.snapshot == "/tmp/s.json" + assert args.backup_path == "/tmp/b.sql.gz" + assert args.backup_max_age_hours == 6.0 + assert args.max_duplicate_groups == 5 + + +def test_postflight_parser_defaults_and_shape(): + args = po.build_parser().parse_args([]) + assert args.database_url is None + assert args.target == "0024" + assert args.json is False + assert args.snapshot is None + assert args.allow_row_growth is False + + +def test_postflight_parser_accepts_the_documented_interface(): + args = po.build_parser().parse_args( + ["--database-url", "postgresql://u:p@h/d", "--target", "0023", + "--json", "--snapshot", "/tmp/s.json", "--allow-row-growth"] + ) + assert args.snapshot == "/tmp/s.json" + assert args.allow_row_growth is True + + +def test_both_scripts_share_one_preflight_module_instance(): + """postflight loads preflight by path; two live copies would give two sets of + dataclasses and two sets of constants that could silently disagree.""" + assert po._pf is pf + + +def test_resolve_database_url_prefers_the_cli_then_the_environment(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://e:e@h:5432/env_db") + assert pf.resolve_database_url("postgresql://c:c@h:5432/cli_db").endswith("/cli_db") + assert pf.resolve_database_url(None).endswith("/env_db") + assert pf.resolve_database_url(None).startswith("postgresql+asyncpg://") + + +def test_write_snapshot_warns_when_no_path_is_given(): + class _Args: + snapshot = None + target = "0023" + + status, detail, remediation = pf.write_snapshot( + _Args(), pf.Report("preflight"), {"users": 1}, "0018" + ) + assert status == pf.WARN + assert "postflight cannot compare" in detail + assert any("--snapshot" in r for r in remediation) + + +def test_write_snapshot_round_trips_through_compare_row_counts(tmp_path): + import json + + class _Args: + target = "0023" + + def __init__(self, path): + self.snapshot = str(path) + + counts = {"users": 3, "agent_messages": 18} + status, detail, _ = pf.write_snapshot( + _Args(tmp_path / "snap.json"), pf.Report("preflight"), counts, "0018" + ) + assert status == pf.PASS + assert "18" in detail or "21" in detail + payload = json.loads((tmp_path / "snap.json").read_text()) + assert payload["kind"] == "preflight-snapshot" + assert payload["current_revision"] == "0018" + assert payload["row_counts"] == counts + ok, problems = pf.compare_row_counts(payload["row_counts"], counts) + assert ok and problems == [] + + +def test_write_snapshot_blocks_when_the_path_is_not_writable(tmp_path): + class _Args: + target = "0023" + snapshot = "/proc/definitely/not/writable/snap.json" + + status, detail, _ = pf.write_snapshot(_Args(), pf.Report("preflight"), {"users": 1}, "0018") + assert status == pf.BLOCK + assert "could not write snapshot" in detail + + +# --------------------------------------------------------------------------- # +# check_ambiguous_revision: THREE files in this repo's history declared 0019 +# +# Enumerated from git rather than memory — every historical blob under +# alembic/versions/ was parsed for its declared revision id: +# +# 0019_agent_message_content.py (a7659b4, this chain) -> agent_messages.content +# 0019_add_cohorts.py (b00b0e6, cohort-agent-isolation) -> cohorts +# 0019_add_hidden_to_proposals.py (4037b79, coPI-podcast) -> thread_decisions.hidden +# +# Why naming the right one matters, measured on fixtures in each state: +# * cohort 0019 -> `alembic upgrade 0023` dies at 0022 with DuplicateTableError, +# revision stays 0019, nothing applied. Loud and safe. +# * podcast 0019 -> `alembic upgrade 0023` EXITS 0 AND STAMPS 0023 while +# agent_messages.content and uq_agent_messages_run_ts do not exist. Alembic +# reports total success on a database the app cannot run against. +# The remediations are opposites (drop the cohort tables vs. leave the two `hidden` +# columns alone), so a check that guessed would send the operator the wrong way. +# --------------------------------------------------------------------------- # + + +def _stub_schema(monkeypatch, *, content: bool, cohorts: bool, hidden: bool) -> None: + async def _column_exists(_conn, table: str, column: str) -> bool: + if (table, column) == ("agent_messages", "content"): + return content + if (table, column) == ("thread_decisions", "hidden"): + return hidden + return False + + async def _table_exists(_conn, name: str) -> bool: + return cohorts if name == "cohorts" else False + + monkeypatch.setattr(pf, "column_exists", _column_exists) + monkeypatch.setattr(pf, "table_exists", _table_exists) + + +async def test_ambiguous_revision_is_not_applicable_away_from_0019(monkeypatch): + _stub_schema(monkeypatch, content=False, cohorts=False, hidden=False) + _title, status, detail, _rem, _data = await pf.check_ambiguous_revision(None, "0018") + assert status == pf.PASS + assert "not applicable" in detail + + +async def test_ambiguous_revision_passes_when_the_content_columns_are_present(monkeypatch): + # The only state that may proceed. Note it passes even alongside the other + # signatures: content present means the right 0019 ran, whatever else is there. + _stub_schema(monkeypatch, content=True, cohorts=True, hidden=True) + _title, status, detail, rem, data = await pf.check_ambiguous_revision(None, "0019") + assert status == pf.PASS + assert "0019_agent_message_content" in detail + assert rem == [] + assert data["agent_messages.content"] is True + + +async def test_ambiguous_revision_names_the_cohort_0019_and_says_to_drop_its_tables(monkeypatch): + _stub_schema(monkeypatch, content=False, cohorts=True, hidden=False) + _title, status, detail, rem, _data = await pf.check_ambiguous_revision(None, "0019") + joined = "\n".join(rem) + assert status == pf.BLOCK + assert "0019_add_cohorts" in detail + assert "0019_add_hidden_to_proposals" not in detail + assert "DROP TABLE IF EXISTS cohort_memberships, cohorts CASCADE;" in joined + # It must warn about the specific way the upgrade fails, so the operator + # recognises the DuplicateTableError when they see it. + assert "0022" in joined + + +async def test_ambiguous_revision_names_the_podcast_0019_and_leaves_its_columns_alone(monkeypatch): + _stub_schema(monkeypatch, content=False, cohorts=False, hidden=True) + _title, status, detail, rem, _data = await pf.check_ambiguous_revision(None, "0019") + joined = "\n".join(rem) + assert status == pf.BLOCK + assert "0019_add_hidden_to_proposals" in detail + assert "0019_add_cohorts" not in detail + # The opposite advice from the cohort case: these columns are orphaned but + # harmless, so the remediation must NOT tell anyone to drop the cohort tables, + # and must prefer leaving data in place. + assert "cohort_memberships" not in joined + assert "left in place" in joined + + +async def test_ambiguous_revision_refuses_to_guess_on_an_unknown_0019(monkeypatch): + _stub_schema(monkeypatch, content=False, cohorts=False, hidden=False) + _title, status, detail, rem, _data = await pf.check_ambiguous_revision(None, "0019") + joined = "\n".join(rem) + assert status == pf.BLOCK + assert "unrecognised 0019" in detail + # Failing closed is not enough: it must not hand over a remediation that was + # written for a different database's history. + assert "inspect the schema by hand" in joined + assert "DROP TABLE" not in joined + + +async def test_ambiguous_revision_distinguishes_all_three_signatures(monkeypatch): + """One assertion that the check actually probes three things, not two. + + Guards the regression where a third 0019 existed but the check only knew about + two, so a podcast-0019 database was correctly blocked and then handed the + cohort remediation. + """ + seen = [] + for cohorts, hidden in ((True, False), (False, True), (False, False)): + _stub_schema(monkeypatch, content=False, cohorts=cohorts, hidden=hidden) + _t, status, detail, _rem, _d = await pf.check_ambiguous_revision(None, "0019") + assert status == pf.BLOCK + seen.append(detail) + assert len(set(seen)) == 3, "each of the three states must be diagnosed differently" diff --git a/tests/unit/test_model_tiering.py b/tests/unit/test_model_tiering.py new file mode 100644 index 0000000..d57badb --- /dev/null +++ b/tests/unit/test_model_tiering.py @@ -0,0 +1,55 @@ +"""Pins the per-phase model tiering. + +The high-volume default-model paths (phase-2 scan/prune, memory synthesis, +and the phase-1 decision helper) run on Sonnet to keep the pre-Opus-5 cost +profile; only the phase-4 reply and phase-5 post paths (which pass +settings.llm_agent_model_opus explicitly) run on Opus 5. A default-model +change silently re-prices every scan/prune/memory call, so the wiring is +pinned here — see PR #30's Opus 5 upgrade, which originally moved the +default from claude-sonnet-4-6 to claude-opus-5 and re-priced those paths +~5x without saying so. +""" + +import pytest + +from src.config import get_settings +from src.services import llm +from tests.fakes import FakeAnthropic, text_response + + +@pytest.fixture(autouse=True) +def _clear_llm_callback(): + llm.set_call_log_callback(None) + yield + llm.set_call_log_callback(None) + + +def test_default_agent_model_is_sonnet_5_and_opus_knob_is_opus_5(): + settings = get_settings() + assert settings.llm_agent_model == "claude-sonnet-5" + assert settings.llm_agent_model_opus == "claude-opus-5" + + +async def test_generate_agent_response_defaults_to_the_sonnet_tier(monkeypatch): + """A model-less call (the phase-2 scan/prune and memory-synthesis shape) + must hit the sonnet-tier default, not Opus.""" + fake = FakeAnthropic([text_response("ok")]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + await llm.generate_agent_response("sys", [{"role": "user", "content": "hi"}]) + + assert fake.calls[0]["model"] == get_settings().llm_agent_model + assert fake.calls[0]["model"] == "claude-sonnet-5" + # The thinking pin must survive the model change: Sonnet 5 runs adaptive + # thinking when the param is omitted, and max_tokens caps thinking + text + # together — the tight per-phase caps rely on this being disabled. + assert fake.calls[0]["thinking"] == {"type": "disabled"} + + +async def test_make_decision_defaults_to_the_sonnet_tier(monkeypatch): + fake = FakeAnthropic([text_response('{"action": "skip"}')]) + monkeypatch.setattr("src.services.llm.get_anthropic_client", lambda: fake) + + await llm.make_decision("sys", [{"role": "user", "content": "hi"}]) + + assert fake.calls[0]["model"] == "claude-sonnet-5" diff --git a/tests/unit/test_post_types.py b/tests/unit/test_post_types.py new file mode 100644 index 0000000..c5d9f98 --- /dev/null +++ b/tests/unit/test_post_types.py @@ -0,0 +1,371 @@ +"""The post-type vocabulary and the role/topology filter. + +Pure functions over plain data — no DB, no engine, no Agent. See +docs/specs/2026-08-06-role-topology-post-type-gating-design.md §2, §3. +""" +import logging + +from src.agent.post_types import ( + CANONICAL, + DEFAULT_POST_TYPES, + FUNDING_POST_TYPES, + LEGACY_POST_TYPE_ALIASES, + available_for, + eligible_targets, + parse_post_types, + render_menu, + resolve_post_type_name, +) + +# The star: a spoke may reach only itself, the hub, and grantbot (which has no +# AgentRegistry row, so no role). +STAR_GATE = {"gill", "blackbird", "grantbot"} +STAR_ROLES = {"gill": "pi_lab", "blackbird": "scout_hub"} +BOT_NAMES = {"gill": "GillBot", "blackbird": "BlackbirdBot", "pearce": "PearceBot"} + +# The mesh: several pi_lab peers, no hub. +MESH_ROLES = {"gill": "pi_lab", "pearce": "pi_lab", "wu": "pi_lab"} + + +def _by_name(specs): + return {s.name for s in specs} + + +def test_canonical_vocabulary_is_exactly_the_spec_table(): + assert set(CANONICAL) == { + "paper", "help_wanted", "introduction", + "idea_crosslab", "pitch", "funding_collab", "opportunity_assessment", + } + + +def test_idea_is_not_a_type_anymore(): + """`idea` and `idea_crosslab` were both in the old enum with no documented + difference and no code distinguishing them. Collapsed to one.""" + assert "idea" not in CANONICAL + assert "idea" not in _by_name(DEFAULT_POST_TYPES) + + +def test_the_retired_idea_name_still_resolves(): + """Retired in the vocabulary, still accepted on input. A mesh deployment + whose prompts lag the code must not have its posts silently deleted.""" + assert resolve_post_type_name("idea") == "idea_crosslab" + + +def test_resolve_passes_current_and_unknown_names_through(): + assert resolve_post_type_name("paper") == "paper" + assert resolve_post_type_name("nonsense") == "nonsense" + + +def test_an_alias_is_never_offered_as_a_type(): + """Resolving on input must not put the retired name back in circulation.""" + for alias in LEGACY_POST_TYPE_ALIASES: + assert alias not in CANONICAL + out = render_menu( + DEFAULT_POST_TYPES, gate=None, roles_by_agent=MESH_ROLES, + self_id="gill", bot_names=BOT_NAMES, + ) + assert f"**`{alias}`**" not in out + + +def test_default_post_types_is_the_pi_lab_set(): + assert _by_name(DEFAULT_POST_TYPES) == { + "paper", "help_wanted", "introduction", + "idea_crosslab", "pitch", "funding_collab", + } + assert "opportunity_assessment" not in _by_name(DEFAULT_POST_TYPES) + + +def test_broadcast_types_carry_no_targets(): + for name in ("paper", "help_wanted", "introduction"): + assert CANONICAL[name].targets == frozenset() + + +def test_addressed_types_declare_their_counterparty_role(): + assert CANONICAL["idea_crosslab"].targets == frozenset({"pi_lab"}) + assert CANONICAL["pitch"].targets == frozenset({"scout_hub"}) + assert CANONICAL["funding_collab"].targets == frozenset({"pi_lab"}) + + +# --- eligible_targets ------------------------------------------------------- + +def test_eligible_targets_excludes_self(): + """An agent's own role is in its own gate; it must never be its own target.""" + spec = CANONICAL["idea_crosslab"] + got = eligible_targets(spec, gate={"gill"}, roles_by_agent={"gill": "pi_lab"}, self_id="gill") + assert got == frozenset() + + +def test_eligible_targets_finds_the_hub_for_pitch(): + got = eligible_targets( + CANONICAL["pitch"], gate=STAR_GATE, roles_by_agent=STAR_ROLES, self_id="gill" + ) + assert got == frozenset({"blackbird"}) + + +def test_eligible_targets_ignores_agents_with_no_known_role(): + """grantbot is in the gate but has no AgentRegistry row and is a separate + process, never an entry in self.agents — so it never appears in + roles_by_agent and matches no `targets`. It is a funding announcer, not a + pitch recipient. Asserted for BOTH addressed types so the exclusion is not + an accident of `pitch` happening to find the hub first.""" + for name in ("pitch", "idea_crosslab", "funding_collab"): + got = eligible_targets( + CANONICAL[name], gate=STAR_GATE, roles_by_agent=STAR_ROLES, self_id="gill" + ) + assert "grantbot" not in got + + +def test_eligible_targets_is_empty_for_a_lab_peer_in_the_star(): + got = eligible_targets( + CANONICAL["idea_crosslab"], gate=STAR_GATE, roles_by_agent=STAR_ROLES, self_id="gill" + ) + assert got == frozenset() + + +def test_eligible_targets_with_gate_off_returns_every_matching_role(): + got = eligible_targets( + CANONICAL["idea_crosslab"], gate=None, roles_by_agent=MESH_ROLES, self_id="gill" + ) + assert got == frozenset({"pearce", "wu"}) + + +# --- available_for ---------------------------------------------------------- + +def test_star_drops_lab_peer_types_and_keeps_pitch(): + got = available_for( + DEFAULT_POST_TYPES, gate=STAR_GATE, roles_by_agent=STAR_ROLES, + self_id="gill", funding_only=False, + ) + assert _by_name(got) == {"paper", "help_wanted", "introduction", "pitch"} + + +def test_mesh_keeps_lab_peer_types_and_drops_pitch(): + got = available_for( + DEFAULT_POST_TYPES, gate=None, roles_by_agent=MESH_ROLES, + self_id="gill", funding_only=False, + ) + assert _by_name(got) == { + "paper", "help_wanted", "introduction", "idea_crosslab", "funding_collab", + } + + +def test_gate_off_keeps_every_broadcast_type(): + got = available_for( + DEFAULT_POST_TYPES, gate=None, roles_by_agent={}, self_id="gill", funding_only=False, + ) + assert {"paper", "help_wanted", "introduction"} <= _by_name(got) + + +def test_funding_only_restricts_to_funding_types(): + got = available_for( + DEFAULT_POST_TYPES, gate=None, roles_by_agent=MESH_ROLES, + self_id="gill", funding_only=True, + ) + assert _by_name(got) == {"funding_collab"} + assert _by_name(got) <= FUNDING_POST_TYPES + + +def test_funding_only_in_the_star_is_empty(): + """Empty is the correct answer here, and the engine must NOT read it as + "skip the turn" — Option A (a funding reply) is still legitimate. That half + is enforced in test_post_type_enforcement.py, not here; this only pins that + the set really is empty. See spec §5.""" + got = available_for( + DEFAULT_POST_TYPES, gate=STAR_GATE, roles_by_agent=STAR_ROLES, + self_id="gill", funding_only=True, + ) + assert got == () + + +def test_available_for_preserves_declaration_order(): + got = available_for( + DEFAULT_POST_TYPES, gate=None, roles_by_agent=MESH_ROLES, + self_id="gill", funding_only=False, + ) + declared = [s.name for s in DEFAULT_POST_TYPES if s.name in _by_name(got)] + assert [s.name for s in got] == declared + + +# --- parse_post_types ------------------------------------------------------- + +def test_parse_none_yields_the_defaults(caplog): + """Spec §5 row 1 says "DEFAULT_POST_TYPES, WARNING once". The defaults are + the correct answer for pi_lab, which HAS no manifest by design, so warning + on every load would be noise on the common path — the warning belongs to a + role that has a manifest and forgot the key. Pinned here so the divergence + from §5 is a decision on record, not a silent omission.""" + caplog.set_level(logging.WARNING) + assert parse_post_types(None, role="pi_lab") == DEFAULT_POST_TYPES + assert caplog.text == "" + + +def test_parse_reads_name_and_targets(): + got = parse_post_types( + [{"name": "opportunity_assessment"}, + {"name": "funding_collab", "targets": ["pi_lab"]}], + role="scout_hub", + ) + assert _by_name(got) == {"opportunity_assessment", "funding_collab"} + assert dict((s.name, s.targets) for s in got)["funding_collab"] == frozenset({"pi_lab"}) + + +def test_parse_drops_an_unknown_name_and_keeps_the_rest(caplog): + got = parse_post_types( + [{"name": "paper"}, {"name": "not_a_real_type"}], role="pi_lab" + ) + assert _by_name(got) == {"paper"} + assert "not_a_real_type" in caplog.text + + +def test_parse_drops_a_malformed_entry_and_keeps_the_rest(caplog): + got = parse_post_types(["paper", {"name": "help_wanted"}, {}], role="pi_lab") + assert _by_name(got) == {"help_wanted"} + assert caplog.text + + +def test_parse_warns_when_targets_names_a_role_that_cannot_exist(caplog): + """A typo'd role means the type is silently never offered — say so at load.""" + caplog.set_level(logging.WARNING) + got = parse_post_types( + [{"name": "pitch", "targets": ["scout_hubb"]}], role="pi_lab" + ) + assert _by_name(got) == {"pitch"} + assert "scout_hubb" in caplog.text + + +def test_a_typod_target_role_really_is_never_offered(caplog): + """The other half of that §5 row. The WARNING is only useful if the + behaviour it predicts is real: no agent can ever satisfy `scout_hubb`, so + the type is filtered out of every menu on every topology.""" + caplog.set_level(logging.WARNING) + declared = parse_post_types( + [{"name": "paper"}, {"name": "pitch", "targets": ["scout_hubb"]}], + role="pi_lab", + ) + for gate, roles in ((STAR_GATE, STAR_ROLES), (None, MESH_ROLES)): + got = available_for( + declared, gate=gate, roles_by_agent=roles, self_id="gill", + funding_only=False, + ) + assert _by_name(got) == {"paper"} + + +def test_parse_of_a_non_list_yields_the_defaults(caplog): + assert parse_post_types("paper", role="pi_lab") == DEFAULT_POST_TYPES + assert caplog.text + + +def test_parse_targets_override_replaces_the_canonical_default(): + got = parse_post_types([{"name": "pitch", "targets": []}], role="pi_lab") + assert got[0].targets == frozenset() + + +# --- render_menu ------------------------------------------------------------ + +def test_render_menu_names_every_available_type_with_its_emoji(): + specs = available_for( + DEFAULT_POST_TYPES, gate=STAR_GATE, roles_by_agent=STAR_ROLES, + self_id="gill", funding_only=False, + ) + out = render_menu( + specs, gate=STAR_GATE, roles_by_agent=STAR_ROLES, self_id="gill", bot_names=BOT_NAMES, + ) + for name in ("paper", "help_wanted", "introduction", "pitch"): + assert CANONICAL[name].emoji in out + assert name in out + assert "idea_crosslab" not in out + + +def test_render_menu_never_prints_an_empty_enumeration(): + """The bug this exists to stop: `Set tagged_agent to exactly one of: .` + + build_phase5_prompt renders a default menu when no caller supplies one, and + test_phase5_prompt_gm goes down that path — so an empty enumeration would be + committed into a characterization snapshot and shipped to a live model. + """ + for gate, roles in ((None, {}), (None, MESH_ROLES), ({"gill"}, {"gill": "pi_lab"})): + out = render_menu( + DEFAULT_POST_TYPES, gate=gate, roles_by_agent=roles, + self_id="gill", bot_names={}, + ) + assert "one of: ." not in out + assert "one of: \n" not in out + assert out.strip() + + +def test_render_menu_does_not_enumerate_when_the_gate_is_off(): + """A mesh has ~50 reachable labs. Enumerating them in every phase-5 prompt + would recreate the 46 KB lab directory this design is shrinking, so gate + None renders guidance instead of a list.""" + out = render_menu( + [CANONICAL["idea_crosslab"]], gate=None, roles_by_agent=MESH_ROLES, + self_id="gill", bot_names=BOT_NAMES, + ) + assert "pearce" not in out and "wu" not in out + assert "pi_lab" in out + assert "agent_id" in out + + +def test_render_menu_enumerates_when_the_gate_is_on(): + out = render_menu( + [CANONICAL["pitch"]], gate=STAR_GATE, roles_by_agent=STAR_ROLES, + self_id="gill", bot_names=BOT_NAMES, + ) + assert "one of:" in out + assert "blackbird" in out + + +def test_render_menu_names_the_reachable_agent_for_an_addressed_type(): + specs = available_for( + DEFAULT_POST_TYPES, gate=STAR_GATE, roles_by_agent=STAR_ROLES, + self_id="gill", funding_only=False, + ) + out = render_menu( + specs, gate=STAR_GATE, roles_by_agent=STAR_ROLES, self_id="gill", bot_names=BOT_NAMES, + ) + assert "BlackbirdBot" in out + assert "blackbird" in out + + +def test_render_menu_marks_a_broadcast_type_as_addressing_no_one(): + out = render_menu( + [CANONICAL["paper"]], gate=STAR_GATE, roles_by_agent=STAR_ROLES, + self_id="gill", bot_names=BOT_NAMES, + ) + assert "no one" in out.lower() or "broadcast" in out.lower() + + +def test_render_menu_of_an_empty_set_says_so_and_points_at_reply_or_skip(): + out = render_menu( + [], gate=STAR_GATE, roles_by_agent=STAR_ROLES, self_id="gill", bot_names=BOT_NAMES, + ) + assert out.strip() + low = out.lower() + assert "no new top-level post type" in low + assert "reply" in low and "skip" in low + + +def test_render_menu_never_returns_an_empty_string(): + """A blank menu would leave the prompt claiming a list exists with nothing in + it, which reads as a rendering bug to the model.""" + for specs in ([], list(DEFAULT_POST_TYPES)): + out = render_menu( + specs, gate=None, roles_by_agent=MESH_ROLES, self_id="gill", bot_names=BOT_NAMES, + ) + assert out.strip() + + +def test_duplicate_post_type_entries_collapse_last_wins(caplog): + """Two [[post_types]] entries for one name must yield ONE spec — the later + one — not two contradictory entries.""" + raw = [ + {"name": "idea_crosslab", "targets": ["pi_lab"]}, + {"name": "paper"}, + {"name": "idea_crosslab", "targets": []}, + ] + with caplog.at_level("WARNING"): + out = parse_post_types(raw, role="probe") + names = [s.name for s in out] + assert names == ["idea_crosslab", "paper"], names # first-occurrence order + assert out[0].targets == frozenset() # last wins + assert any("duplicate post_types entry" in r.message for r in caplog.records) diff --git a/tests/unit/test_private_channel_migration.py b/tests/unit/test_private_channel_migration.py index 3621709..e59ea39 100644 --- a/tests/unit/test_private_channel_migration.py +++ b/tests/unit/test_private_channel_migration.py @@ -209,15 +209,16 @@ def test_returns_mock_channel_with_is_private(self, mock_client): # Mock mode applies the same timestamp suffix as the live path. assert ch["name"].startswith("priv-test-") assert ch["is_private"] is True - assert ch["id"].startswith("mock_priv_") + # Slack-off channels use the DB-native 'local:' id scheme. + assert ch["id"].startswith("local:") def test_public_create_channel_still_works(self, mock_client): """Don't regress the existing create_channel behavior.""" ch = mock_client.create_channel("general") assert ch is not None assert ch["name"] == "general" - # Mock public channels use the 'mock_' prefix (no 'priv_'). - assert ch["id"] == "mock_general" + # Slack-off channels use the DB-native 'local:' id scheme. + assert ch["id"] == "local:general" class _FakeSlack: diff --git a/tests/unit/test_reachability.py b/tests/unit/test_reachability.py new file mode 100644 index 0000000..2ba49f6 --- /dev/null +++ b/tests/unit/test_reachability.py @@ -0,0 +1,1151 @@ +"""Reachability gate: nothing in this repo asserted that routes, templates and +imports are actually *reachable*, and four live defects grew in that blind spot. +All four are now repaired, so the ``KNOWN_*`` suppression sets below are empty and +this file has no strict xfails left: every finding is a real failure. + +What "reachable" means here, precisely: + + * A **template** is reachable if a handler in ``src/`` renders it by name, or if a + reachable template ``extends``/``include``s/``import``s it. Reachability is + transitive, which is the whole point: a link inside an orphaned template must not + launder the route it points at into "referenced". + * A **route** is reachable if a *reachable* template links to it (``href``/``action``/ + ``location.href``/``fetch``), or a string in ``src/`` names it (redirect target, + email link, Slack message), or it is on ``ROUTE_ALLOWLIST`` with a reason. + * An **import** is live if it resolves. Every ``from src.* import name`` in ``src/`` + is checked, plus every import nested in a ``try``. A lazy import inside a function + body is never exercised until that branch runs, and when the branch is wrapped in + ``except Exception: pass`` a stale symbol is invisible forever. + +Design notes that keep this gate from crying wolf (a noisy gate gets deleted): + + * Two matchers, deliberately asymmetric. ``_link_can_reach`` is permissive — a Jinja + expression is allowed to stand in for a literal route segment — so we never call a + working link broken. ``_link_credits`` is strict — a Jinja expression only fills a + ``{path_param}`` slot — so a route only counts as referenced by a link that really + addresses it. Being permissive in one direction and strict in the other trades + false positives (which get the gate deleted) for false negatives (which just leave + a future orphan for the next reader). + * Broken links are only reported for *reachable* templates. Deleting one orphaned + template would otherwise cascade into a pile of derived findings. + * ``ROUTE_ALLOWLIST`` entries carry a written reason and are themselves gated: + ``test_route_allowlist_has_no_stale_entries`` fails if an allowlisted route becomes + referenced. A stale suppression is the same bug wearing a disguise. + +Known false negative, recorded because it is not hypothetical. This gate is static: a +link counts as a credit if it appears in a reachable template, and nothing here +evaluates the Jinja condition the link sits under. A control behind a branch that never +holds is therefore invisible to it. There is a live instance: +``POST /onboarding/retry`` (src/routers/onboarding.py:317) has exactly one control in +the app — the "Try Again" form at templates/onboarding/profile_review.html:53 — and it +sits inside ``{% elif job_status == 'failed' %}``. ``job_status`` is +``Job.status``, and src/worker/main.py only ever writes 'processing', 'completed', +'dead' or 'pending'; ``'failed'`` is permitted by the enum (src/models/job.py:23) and +assigned by nothing in src/. So the retry button is unreachable at runtime while this +gate reports the route as referenced. Closing it would mean evaluating template +conditions against the values src/ can actually produce — a different and much larger +analysis, and one that would cry wolf. Left as a false negative on purpose (the same +trade recorded above: false negatives leave a future orphan, false positives get the +gate deleted), but recorded so the next reader does not mistake this gate's silence for +proof that every control is live. + * The ``KNOWN_*`` sets are the escape hatch, and they are empty. While a defect was + live its entry was subtracted from the aggregate assertions (so those stayed green + and still failed loudly on a *new* orphan) and it carried a paired + ``xfail(strict=True)`` test asserting the defect was fixed — which is what turned + this file red the moment it *was* fixed, forcing the entry out. That mechanism is + still wired up (``test_every_known_defect_entry_is_paired_with_a_strict_xfail``) + and is the only sanctioned way to record a finding you are not fixing today. It + was chosen over plain characterization asserts: an equality assert on today's + broken value passes forever and never notices the repair. +""" + +from __future__ import annotations + +import ast +import functools +import importlib +import re +from dataclasses import dataclass +from pathlib import Path + +import src +from src.main import create_app + +REPO_ROOT = Path(__file__).resolve().parents[2] +SRC_DIR = REPO_ROOT / "src" +TEMPLATES_DIR = REPO_ROOT / "templates" +STATIC_DIR = REPO_ROOT / "static" + +# Stand-in for "a Jinja expression / f-string hole was here". A NUL byte cannot occur +# in a URL or a template, so it can never collide with real content. +HOLE = "\x00" + + +# --------------------------------------------------------------------------- +# Known-live defects: none. All four are repaired, so every set here is empty and the +# aggregate gates below subtract nothing. +# +# Do NOT add to these to silence a new finding — fix the finding. If you genuinely +# cannot fix it today, an entry is subtracted from its aggregate gate and MUST come +# with a paired xfail(strict=True) test asserting the defect is fixed, so the repair +# turns this file red and forces the entry back out. That pairing is enforced by +# test_every_known_defect_entry_is_paired_with_a_strict_xfail (and you will need to +# re-add `import pytest`, dropped when the last defect test went). +# --------------------------------------------------------------------------- + +KNOWN_ORPHAN_TEMPLATES: set[str] = set() + +KNOWN_UNREACHABLE_ROUTES: set[tuple[str, str]] = set() + +KNOWN_BROKEN_LINKS: set[tuple[str, str, str]] = set() + +KNOWN_DEAD_IMPORTS: set[tuple[str, str]] = set() + + +# --------------------------------------------------------------------------- +# Allowlist: routes that are legitimately referenced by neither a template nor src/. +# Every entry needs a reason naming the real caller. Kept honest by +# test_route_allowlist_has_no_stale_entries. +# --------------------------------------------------------------------------- + +ROUTE_ALLOWLIST: dict[tuple[str, str], str] = { + ("GET", "/docs"): "FastAPI-generated Swagger UI; entered by typing the URL.", + ("GET", "/docs/oauth2-redirect"): "FastAPI-generated; used by Swagger UI's own JS.", + ("GET", "/redoc"): "FastAPI-generated ReDoc UI; entered by typing the URL.", + ("GET", "/openapi.json"): "FastAPI-generated schema; fetched by /docs and /redoc JS.", + ("GET", "/api/health"): ( + "Liveness probe, not a page link: docker-compose/nginx and any uptime check " + "hit it directly. Defined inline in src/main.py create_app()." + ), + ("GET", "/admin"): ( + "Bare-URL alias an admin types by hand: src/routers/admin.py stacks " + "@router.get('') and @router.get('/users') on the same admin_users handler, and " + "the nav links the /admin/... children rather than the bare prefix. Nothing to " + "reach — deleting it would only break typed URLs and bookmarks." + ), + ("GET", "/auth/callback"): ( + "ORCID OAuth redirect_uri — the caller is orcid.org. The value we register with " + "ORCID is settings.orcid_redirect_uri (src/config.py), not a link in this app." + ), + ("GET", "/cabo-graph"): ( + "Public collaboration-graph page shared by URL (retreat handout / email), " + "deliberately unlinked from the nav. nginx/nginx.conf:111 whitelists it " + "alongside the other three graph URLs, which is the external caller." + ), + ("GET", "/scripps-graph"): ( + "Same as /cabo-graph: hand-shared public graph URL, whitelisted in " + "nginx/nginx.conf:111, intentionally not in the nav." + ), + ("GET", "/schultz-alumni-pilot"): ( + "Same as /cabo-graph: hand-shared public graph URL for the Schultz alumni " + "pilot cohort, whitelisted in nginx/nginx.conf:111." + ), + ("GET", "/schultz-group-alumni"): ( + "Same as /cabo-graph: hand-shared public graph URL for the Schultz group " + "alumni cohort, whitelisted in nginx/nginx.conf:111." + ), +} + +# Optional third-party imports that are allowed to be absent at test time. Empty +# today: every `try:`-guarded import in src/ resolves in the app image. +GUARDED_IMPORT_ALLOWLIST: dict[str, str] = {} + +# Targets of any template reference we cannot resolve statically (a dynamic +# `{% include some_var %}`). Empty today — test_no_dynamic_template_references keeps it +# that way, because a dynamic include is a hole the orphan-template gate cannot see +# through. If one is genuinely needed, list its possible targets here. +DYNAMIC_TEMPLATE_TARGETS: set[str] = set() + + +# --------------------------------------------------------------------------- +# Route table +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Route: + method: str + path: str + name: str + + @property + def key(self) -> tuple[str, str]: + return (self.method, self.path) + + +def _walk_routes(routes, prefix: str = "") -> list[Route]: + """Flatten the assembled app's route table into (method, full path) pairs. + + FastAPI >= 0.140 stores an included router as a ``_IncludedRouter`` wrapper that + keeps the original router plus its mount prefix, so ``app.routes`` is a tree, not a + list. Older versions nest via ``.routes``. Both shapes are handled; ``Mount``s + (``/static``) are recorded as opaque prefixes rather than descended into. + """ + out: list[Route] = [] + for route in routes: + ctx = getattr(route, "include_context", None) + if ctx is not None: + out += _walk_routes(ctx.included_router.routes, prefix + (ctx.prefix or "")) + continue + path = getattr(route, "path", None) + if path is None: + continue + sub = getattr(route, "routes", None) + if sub is not None and type(route).__name__ == "Mount": + out.append(Route("MOUNT", prefix + path, getattr(route, "name", "") or "")) + continue + if sub is not None: + out += _walk_routes(sub, prefix + path) + continue + for method in sorted(getattr(route, "methods", None) or ()): + out.append(Route(method, prefix + path, getattr(route, "name", "") or "")) + return out + + +@functools.lru_cache(maxsize=1) +def route_table() -> tuple[Route, ...]: + return tuple(_walk_routes(create_app().routes)) + + +@functools.lru_cache(maxsize=1) +def http_routes() -> tuple[Route, ...]: + """Routes a browser can address. HEAD is dropped: Starlette adds it alongside GET, + so treating it separately would double every finding.""" + return tuple(r for r in route_table() if r.method not in {"MOUNT", "HEAD"}) + + +# --------------------------------------------------------------------------- +# Path matching +# --------------------------------------------------------------------------- + + +def _segments(path: str) -> list[str]: + stripped = path.strip("/") + return stripped.split("/") if stripped else [] + + +def _is_param(segment: str) -> bool: + return segment.startswith("{") and segment.endswith("}") + + +def _link_can_reach(link_path: str, route_path: str) -> bool: + """Permissive: could this link ever hit this route? Used to decide whether a link + is broken, so a Jinja expression is allowed to render into a literal segment.""" + link_segs, route_segs = _segments(link_path), _segments(route_path) + if len(link_segs) != len(route_segs): + return False + for link_seg, route_seg in zip(link_segs, route_segs, strict=True): + if _is_param(route_seg) or link_seg == route_seg or HOLE in link_seg: + continue + return False + return True + + +def _link_credits(link_path: str, route_path: str) -> bool: + """Strict: does this link actually address this route? Used to decide whether a + route is referenced, so a Jinja expression may only fill a ``{path_param}``.""" + link_segs, route_segs = _segments(link_path), _segments(route_path) + if len(link_segs) != len(route_segs): + return False + for link_seg, route_seg in zip(link_segs, route_segs, strict=True): + if _is_param(route_seg): + continue + if link_seg == route_seg: + continue + return False + return True + + +# --------------------------------------------------------------------------- +# Template graph +# --------------------------------------------------------------------------- + + +@functools.lru_cache(maxsize=1) +def template_names() -> tuple[str, ...]: + """Every file under templates/, named the way Jinja2Templates(directory=...) does.""" + return tuple( + sorted( + p.relative_to(TEMPLATES_DIR).as_posix() + for p in TEMPLATES_DIR.rglob("*") + if p.is_file() + ) + ) + + +# {% extends "x" %} / {% include "x" %} / {% import "x" as y %} / {% from "x" import y %} +_TEMPLATE_REF_RE = re.compile( + r"{%-?\s*(extends|include|import|from)\s+(?P<rest>.+?)\s*-?%}", re.S +) +_QUOTED_RE = re.compile(r"""(?:"([^"]*)"|'([^']*)')""") + + +@functools.lru_cache(maxsize=1) +def template_to_template_refs() -> dict[str, frozenset[str]]: + """parent template -> templates it pulls in (literal names only).""" + refs: dict[str, frozenset[str]] = {} + for name in template_names(): + text = (TEMPLATES_DIR / name).read_text(encoding="utf-8", errors="replace") + found: set[str] = set() + for m in _TEMPLATE_REF_RE.finditer(text): + rest = m.group("rest") + # `{% from "x" import y %}` — only the leading quoted token is the template. + head = rest.split(" import ")[0] if m.group(1) == "from" else rest + found.update(q for pair in _QUOTED_RE.findall(head) for q in pair if q) + refs[name] = frozenset(found) + return refs + + +@functools.lru_cache(maxsize=1) +def dynamic_template_refs() -> tuple[tuple[str, int, str], ...]: + """(template, line, directive) for every include/extends whose target is not a + quoted literal — a hole this gate cannot see through.""" + out: list[tuple[str, int, str]] = [] + for name in template_names(): + text = (TEMPLATES_DIR / name).read_text(encoding="utf-8", errors="replace") + for m in _TEMPLATE_REF_RE.finditer(text): + rest = m.group("rest") + head = rest.split(" import ")[0] if m.group(1) == "from" else rest + if not _QUOTED_RE.search(head): + out.append((name, text[: m.start()].count("\n") + 1, m.group(0).strip())) + return tuple(out) + + +# --------------------------------------------------------------------------- +# src/ scan: rendered template names, path-ish strings, imports +# --------------------------------------------------------------------------- + + +@functools.lru_cache(maxsize=1) +def _src_files() -> tuple[Path, ...]: + return tuple(sorted(p for p in SRC_DIR.rglob("*.py") if "__pycache__" not in p.parts)) + + +def _flatten_fstring(node: ast.AST) -> str | None: + """Reconstruct a str constant or f-string, with HOLE for each interpolation. + Returns None for anything that is not a string expression.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr): + parts = [] + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + parts.append(value.value) + else: + parts.append(HOLE) + return "".join(parts) + return None + + +_HTTP_DECORATORS = { + "get", "post", "put", "patch", "delete", "head", "options", "trace", + "api_route", "route", "websocket", +} + + +def _excluded_string_nodes(tree: ast.AST) -> set[int]: + """Strings that mention a path without *calling* it, and must not credit a route. + + Two kinds, and both matter: + + * A route decorator's own path (``@router.get("/auth/callback")``). Routers + mounted without a prefix declare their full path there, so without this every + such route would credit itself and the unreachable-route gate would be blind + to exactly half the app. (Prefixed routers accidentally escape that, which is + the only reason the /onboarding orphans were visible at all.) + * A router's own mount ``prefix=``. ``include_router(admin.router, + prefix="/admin")`` is the definition of ``GET /admin``, not a link to it. + * Docstrings. "``/login, /auth/callback, /logout``" in a module docstring is + documentation, not a caller. + """ + excluded: set[int] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Call): + for kw in node.keywords: + if kw.arg == "prefix": + excluded.add(id(kw.value)) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for dec in node.decorator_list: + if ( + isinstance(dec, ast.Call) + and isinstance(dec.func, ast.Attribute) + and dec.func.attr in _HTTP_DECORATORS + and dec.args + ): + excluded.add(id(dec.args[0])) + if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + body = getattr(node, "body", []) + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + excluded.add(id(body[0].value)) + return excluded + + +@functools.lru_cache(maxsize=1) +def src_strings() -> tuple[tuple[str, str], ...]: + """(file, reconstructed string) for every string literal / f-string in src/, minus + route-decorator paths and docstrings (see _excluded_string_nodes).""" + out: list[tuple[str, str]] = [] + for path in _src_files(): + rel = path.relative_to(REPO_ROOT).as_posix() + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + excluded = _excluded_string_nodes(tree) + for node in ast.walk(tree): + if isinstance(node, (ast.Constant, ast.JoinedStr)) and id(node) not in excluded: + text = _flatten_fstring(node) + if text is not None: + out.append((rel, text)) + return tuple(out) + + +@functools.lru_cache(maxsize=1) +def templates_rendered_by_src() -> frozenset[str]: + known = set(template_names()) + return frozenset(text for _, text in src_strings() if text in known) + + +@functools.lru_cache(maxsize=1) +def dynamic_template_names_in_src() -> tuple[tuple[str, str], ...]: + """f-strings in src/ that look like a computed template name — another blind spot.""" + return tuple( + (rel, text) + for rel, text in src_strings() + if HOLE in text and text.endswith(".html") + ) + + +_PATH_TOKEN_SPLIT = re.compile(r"""[\s"'`<>()\[\]{},;|\\]+""") + + +def _candidate_paths(text: str) -> set[str]: + """Pull every URL path out of an arbitrary string (redirect target, email body, + Slack message). Query and fragment are dropped; ``{}`` are kept only when they came + from an f-string hole, which ``_flatten_fstring`` already turned into HOLE.""" + out: set[str] = set() + for token in _PATH_TOKEN_SPLIT.split(text): + if "/" not in token: + continue + candidate = token[token.index("/") :] + candidate = candidate.split("?", 1)[0].split("#", 1)[0] + if not candidate.startswith("/") or candidate == "/": + out.add("/") if candidate == "/" else None + continue + out.add(candidate.rstrip("/") or "/") + if text.strip() == "/": + out.add("/") + return out + + +@functools.lru_cache(maxsize=1) +def src_referenced_paths() -> frozenset[str]: + """Paths named by any non-docstring, non-decorator string in src/: redirect targets, + email links, Slack message bodies.""" + out: set[str] = set() + for _, text in src_strings(): + out |= _candidate_paths(text) + return frozenset(out) + + +# --------------------------------------------------------------------------- +# Template links +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Link: + template: str + method: str + raw: str + path: str # normalized: query/fragment stripped, Jinja expressions -> HOLE + # False for a URL found in JavaScript, where the verb lives in an options object we + # do not parse. Such a link still *credits* a route (any verb), but is never used to + # call a link broken — we would only be guessing at the method. + method_known: bool = True + + +_FORM_TAG_RE = re.compile(r"<form\b[^>]*>", re.I | re.S) +_ANCHOR_ATTR_RE = re.compile(r"""\b(?:href)\s*=\s*(?:"([^"]*)"|'([^']*)')""", re.I) +_ACTION_ATTR_RE = re.compile(r"""\baction\s*=\s*(?:"([^"]*)"|'([^']*)')""", re.I) +_METHOD_ATTR_RE = re.compile(r"""\bmethod\s*=\s*(?:"([^"]*)"|'([^']*)')""", re.I) +# Client-side navigation: always a GET. +_JS_NAV_RE = re.compile( + r"""(?:location\.href\s*=|location\.assign\(|location\.replace\(|window\.open\()\s*""" + r"""(?:"([^"]*)"|'([^']*)')""", + re.I, +) +_SCRIPT_BLOCK_RE = re.compile(r"<script\b[^>]*>(.*?)</script>", re.I | re.S) +# `"/a/" + id + "/b"` -> `"/a/\x00/b"`, so a URL assembled by concatenation still +# resolves to a path pattern. Without this, every JS-called API with an id in the middle +# would look unreachable. +_JS_CONCAT_RE = re.compile(r"""["']\s*\+\s*[^+"'()]{1,60}\+\s*["']""") +_JS_STRING_RE = re.compile(r"""(?:"([^"\n]*)"|'([^'\n]*)')""") +_JINJA_EXPR_RE = re.compile(r"{{.*?}}", re.S) +_JINJA_TAG_RE = re.compile(r"{%.*?%}", re.S) + +_EXTERNAL_PREFIXES = ("http://", "https://", "//", "mailto:", "tel:", "javascript:", "data:") + + +def _normalize_link(raw: str) -> list[str]: + """Turn one attribute value into zero or more candidate route paths. + + A value may hold several: ``{% if x %}/a{% else %}/b{% endif %}`` really is two + links, and both should resolve. Statement tags split the value into chunks; each + chunk is then classified. Anything we cannot pin to a local path (an external URL, + a bare ``{{ var }}``, an anchor, a same-page ``?query=`` link) yields nothing — + those are counted as unresolvable rather than guessed at. + """ + out: list[str] = [] + for chunk in _JINJA_TAG_RE.split(raw): + value = chunk.strip() + if not value or value.startswith(("#", "?")): + continue + if value.lower().startswith(_EXTERNAL_PREFIXES): + continue + value = value.split("#", 1)[0].split("?", 1)[0] + value = _JINJA_EXPR_RE.sub(HOLE, value).strip() + if not value.startswith("/"): + continue + if value.startswith("/static/"): + continue + out.append(value.rstrip("/") or "/") + return out + + +@functools.lru_cache(maxsize=1) +def template_links() -> tuple[Link, ...]: + """Every navigable target in every template, with the HTTP method it will use.""" + links: list[Link] = [] + for name in template_names(): + if not name.endswith((".html", ".htm", ".jinja", ".j2")): + continue + text = (TEMPLATES_DIR / name).read_text(encoding="utf-8", errors="replace") + + # Forms first, so their action= is attributed to the declared method. + form_spans: list[tuple[int, int]] = [] + for tag in _FORM_TAG_RE.finditer(text): + form_spans.append(tag.span()) + action = _ACTION_ATTR_RE.search(tag.group(0)) + if action is None: + continue # no action -> submits to the current URL + method_m = _METHOD_ATTR_RE.search(tag.group(0)) + method = (method_m.group(1) or method_m.group(2)).upper() if method_m else "GET" + raw = action.group(1) if action.group(1) is not None else action.group(2) + for path in _normalize_link(raw): + links.append(Link(name, method, raw, path)) + + for m in _ANCHOR_ATTR_RE.finditer(text): + if any(start <= m.start() < end for start, end in form_spans): + continue # href inside a <form ...> tag is not a navigation target + raw = m.group(1) if m.group(1) is not None else m.group(2) + for path in _normalize_link(raw): + links.append(Link(name, "GET", raw, path)) + + for m in _JS_NAV_RE.finditer(text): + raw = m.group(1) if m.group(1) is not None else m.group(2) + for path in _normalize_link(raw): + links.append(Link(name, "GET", raw, path)) + + # Any local-looking path in a <script> block: an API the page's JS calls. The + # verb is unknown, so these only ever credit a route. + for block in _SCRIPT_BLOCK_RE.finditer(text): + body = _JS_CONCAT_RE.sub(HOLE, block.group(1)) + for m in _JS_STRING_RE.finditer(body): + raw = m.group(1) if m.group(1) is not None else m.group(2) + for path in _normalize_link(raw): + for method in ("GET", "POST", "PUT", "PATCH", "DELETE"): + links.append(Link(name, method, raw, path, method_known=False)) + return tuple(links) + + +@functools.lru_cache(maxsize=1) +def static_js_paths() -> frozenset[str]: + """Local paths named by shipped JavaScript under static/ — same blind spot as an + inline <script>, same treatment.""" + out: set[str] = set() + if not STATIC_DIR.exists(): + return frozenset() + for path in STATIC_DIR.rglob("*.js"): + body = _JS_CONCAT_RE.sub(HOLE, path.read_text(encoding="utf-8", errors="replace")) + for m in _JS_STRING_RE.finditer(body): + raw = m.group(1) if m.group(1) is not None else m.group(2) + out.update(_normalize_link(raw)) + return frozenset(out) + + +@functools.lru_cache(maxsize=1) +def link_attr_values() -> tuple[tuple[str, str], ...]: + """(template, raw value) for every href/action, resolvable or not — the denominator + for the "what fraction can we check" number.""" + out: list[tuple[str, str]] = [] + attr_re = re.compile(r"""\b(?:href|action)\s*=\s*(?:"([^"]*)"|'([^']*)')""", re.I) + for name in template_names(): + if not name.endswith((".html", ".htm", ".jinja", ".j2")): + continue + text = (TEMPLATES_DIR / name).read_text(encoding="utf-8", errors="replace") + for m in attr_re.finditer(text): + out.append((name, m.group(1) if m.group(1) is not None else m.group(2))) + return tuple(out) + + +# --------------------------------------------------------------------------- +# Reachability closure +# --------------------------------------------------------------------------- + + +def compute_reachable_templates( + rendered_by_src: frozenset[str], + refs: dict[str, frozenset[str]], + extra_roots: set[str] | None = None, +) -> set[str]: + """Transitive closure from the templates src/ renders, over extends/include.""" + reachable = set(rendered_by_src) | set(extra_roots or ()) + frontier = list(reachable) + while frontier: + current = frontier.pop() + for child in refs.get(current, ()): # include/extends/import targets + if child not in reachable: + reachable.add(child) + frontier.append(child) + return reachable + + +@functools.lru_cache(maxsize=1) +def reachable_templates() -> frozenset[str]: + return frozenset( + compute_reachable_templates( + templates_rendered_by_src(), + template_to_template_refs(), + extra_roots=set(DYNAMIC_TEMPLATE_TARGETS), + ) + ) + + +def compute_orphan_templates(all_names, reachable) -> set[str]: + return { + name + for name in all_names + if name not in reachable and name.endswith((".html", ".htm", ".jinja", ".j2")) + } + + +def compute_broken_links(links, routes, reachable) -> set[tuple[str, str, str]]: + """Links in *reachable* templates whose (method, path) hits no route.""" + broken = set() + for link in links: + if link.template not in reachable or not link.method_known: + continue + if any( + link.method == r.method and _link_can_reach(link.path, r.path) + for r in routes + ): + continue + broken.add((link.template, link.method, link.path)) + return broken + + +def compute_stale_allowlist_entries( + routes, links, reachable, src_paths, allowlist +) -> dict[tuple[str, str], str]: + """Allowlist entries that no longer suppress anything: the route was deleted, or a + real caller appeared. Either way the entry must go, or it will keep hiding the next + orphan that lands on the same path.""" + unreachable_without_allowlist = compute_unreachable_routes( + routes, links, reachable, src_paths, allowlist={} + ) + registered = {r.key for r in routes} + stale: dict[tuple[str, str], str] = {} + for key in allowlist: + if key not in registered: + stale[key] = "route no longer exists; delete the entry" + elif key not in unreachable_without_allowlist: + stale[key] = "now referenced by a template or src/; delete the entry" + return stale + + +def compute_unreachable_routes( + routes, links, reachable, src_paths, allowlist +) -> set[tuple[str, str]]: + """Routes credited by no reachable template, no src/ string, and no allowlist.""" + live_links = [link for link in links if link.template in reachable] + unreachable = set() + for route in routes: + if route.key in allowlist: + continue + if any( + link.method == route.method and _link_credits(link.path, route.path) + for link in live_links + ): + continue + if any(_link_credits(path, route.path) for path in src_paths): + continue + unreachable.add(route.key) + return unreachable + + +# --------------------------------------------------------------------------- +# Import resolution +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ImportSite: + file: str + line: int + source: str + module: str + names: tuple[str, ...] + in_try: bool + + +def _resolve_relative(path: Path, level: int, module: str | None) -> str: + """Turn `from ..x import y` into an absolute dotted module name. + + ``level`` is 1 for ``from .x``, 2 for ``from ..x``. Level 1 is relative to the + file's own package, so a file at src/services/foo.py has package ``src.services``. + """ + parts = list(path.relative_to(REPO_ROOT).with_suffix("").parts) + if parts and parts[-1] == "__init__": + parts.pop() + package = parts[:-1] if parts else [] + base = package[: len(package) - (level - 1)] if level > 1 else package + return ".".join([*base, *([module] if module else [])]) + + +@functools.lru_cache(maxsize=1) +def import_sites() -> tuple[ImportSite, ...]: + """Every `from ... import ...` / `import ...` in src/, flagged if nested in a try.""" + sites: list[ImportSite] = [] + for path in _src_files(): + rel = path.relative_to(REPO_ROOT).as_posix() + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + guarded: set[int] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Try): + for stmt in node.body: + for sub in ast.walk(stmt): + guarded.add(id(sub)) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + module = ( + _resolve_relative(path, node.level, node.module) + if node.level + else (node.module or "") + ) + sites.append( + ImportSite( + rel, + node.lineno, + ast.unparse(node), + module, + tuple(a.name for a in node.names), + id(node) in guarded, + ) + ) + elif isinstance(node, ast.Import): + for alias in node.names: + sites.append( + ImportSite( + rel, + node.lineno, + ast.unparse(node), + alias.name, + (), + id(node) in guarded, + ) + ) + return tuple(sites) + + +def resolve_import(site: ImportSite) -> str | None: + """None if the import resolves, else a human-readable reason.""" + try: + module = importlib.import_module(site.module) + except Exception as exc: # ImportError, or anything raised at module import time + return f"cannot import {site.module!r}: {type(exc).__name__}: {exc}" + for name in site.names: + if name == "*" or hasattr(module, name): + continue + try: + importlib.import_module(f"{site.module}.{name}") + except Exception: + return f"{site.module!r} has no attribute {name!r}" + return None + + +def compute_dead_imports(sites, allowlist) -> set[tuple[str, str, str]]: + """(file, source, reason) for first-party or try-guarded imports that don't resolve.""" + dead = set() + for site in sites: + first_party = site.module == "src" or site.module.startswith("src.") + if not first_party and not site.in_try: + continue # a plain third-party import failing would break collection anyway + if not first_party and site.module.split(".")[0] in allowlist: + continue + reason = resolve_import(site) + if reason: + dead.add((site.file, site.source, reason)) + return dead + + +# =========================================================================== +# Tests +# =========================================================================== + + +def test_gate_analyzes_the_repo_checkout_not_an_installed_copy(): + """The app image carries a stale `src` in site-packages; if `import src` picked that + up, every finding below would describe code nobody edits. Fail loudly instead.""" + imported = Path(src.__file__).resolve().parent + assert imported == SRC_DIR, ( + f"`import src` resolved to {imported}, not the checkout at {SRC_DIR}. " + "Run pytest from the repo root so the working tree wins over site-packages." + ) + + +def test_route_table_is_fully_enumerated(): + """A route-table walker that silently misses a nested router would make every + 'unreachable route' finding meaningless. Cross-check against the OpenAPI schema, + which FastAPI builds by its own traversal.""" + app = create_app() + from_openapi = set(app.openapi()["paths"]) + from_walk = {r.path for r in http_routes()} + missing = from_openapi - from_walk + assert not missing, f"walker missed routes that OpenAPI knows about: {sorted(missing)}" + assert len(http_routes()) > 50, f"suspiciously few routes: {len(http_routes())}" + + +def test_no_dynamic_template_references(): + """`{% include some_var %}` is a hole the orphan gate cannot see through. If one is + added, list its possible targets in DYNAMIC_TEMPLATE_TARGETS.""" + assert not dynamic_template_refs(), ( + "template pulled in by a computed name — the orphan-template gate cannot " + "follow it. Add its possible targets to DYNAMIC_TEMPLATE_TARGETS:\n" + + "\n".join(f" {t}:{ln} {d}" for t, ln, d in dynamic_template_refs()) + ) + assert not dynamic_template_names_in_src(), ( + "src/ renders a computed template name; the gate cannot follow it:\n" + + "\n".join(f" {rel} {text!r}" for rel, text in dynamic_template_names_in_src()) + ) + + +def test_no_unreferenced_templates(): + """Every template is rendered by a handler or pulled in by a reachable template.""" + orphans = compute_orphan_templates(template_names(), reachable_templates()) + assert orphans - KNOWN_ORPHAN_TEMPLATES == set(), ( + "template referenced by no route and no other template:\n" + + "\n".join(f" templates/{n}" for n in sorted(orphans - KNOWN_ORPHAN_TEMPLATES)) + ) + + +def test_template_links_resolve_to_a_real_route(): + """Every href/action in a reachable template hits a registered (method, path).""" + broken = compute_broken_links(template_links(), http_routes(), reachable_templates()) + assert broken - KNOWN_BROKEN_LINKS == set(), ( + "link/form action resolving to no route:\n" + + "\n".join( + f" templates/{t}: {m} {p}" for t, m, p in sorted(broken - KNOWN_BROKEN_LINKS) + ) + ) + + +def test_no_unreachable_routes(): + """Every registered route is addressed by a reachable template, a src/ string, or + an allowlist entry with a reason.""" + unreachable = compute_unreachable_routes( + http_routes(), + template_links(), + reachable_templates(), + src_referenced_paths() | static_js_paths(), + ROUTE_ALLOWLIST, + ) + assert unreachable - KNOWN_UNREACHABLE_ROUTES == set(), ( + "route referenced by no template and no src/ string. If a caller exists that " + "this gate cannot see (JS, an OAuth callback, an email link), add it to " + "ROUTE_ALLOWLIST with the reason:\n" + + "\n".join( + f" {m} {p}" for m, p in sorted(unreachable - KNOWN_UNREACHABLE_ROUTES) + ) + ) + + +def test_route_allowlist_has_no_stale_entries(): + """A suppression that is no longer needed is the same bug in a different place.""" + stale = compute_stale_allowlist_entries( + http_routes(), + template_links(), + reachable_templates(), + src_referenced_paths() | static_js_paths(), + ROUTE_ALLOWLIST, + ) + assert not stale, "ROUTE_ALLOWLIST entries that are no longer needed:\n" + "\n".join( + f" {m} {p} — {why}" for (m, p), why in sorted(stale.items()) + ) + + +def test_every_allowlist_entry_has_a_reason(): + for key, reason in ROUTE_ALLOWLIST.items(): + assert reason and len(reason) > 20, f"{key} needs a real reason, got {reason!r}" + + +def test_every_known_defect_entry_is_paired_with_a_strict_xfail(): + """The ``KNOWN_*`` sets are suppressions: each subtracts a finding from an aggregate + gate. The docstring says "Do NOT add to this list to silence a new finding", and + until now nothing enforced it — a sixth entry would have gone in silently and the + gate would have gone quiet with it. + + What makes a ``KNOWN_*`` entry legitimate is the paired ``xfail(strict=True)`` test: + that is what turns the file red the moment the defect is repaired, forcing the entry + out. So the invariant is a one-to-one count. Adding an entry without a paired + defect test fails here; deleting a defect test but leaving its entry behind fails + here too. + + Deliberately a count and not a name-matching scheme: a mapping keyed on test names + would itself need maintaining, and the thing worth protecting is that the two never + drift apart in size. + """ + entries = ( + [("KNOWN_ORPHAN_TEMPLATES", e) for e in KNOWN_ORPHAN_TEMPLATES] + + [("KNOWN_UNREACHABLE_ROUTES", e) for e in KNOWN_UNREACHABLE_ROUTES] + + [("KNOWN_BROKEN_LINKS", e) for e in KNOWN_BROKEN_LINKS] + + [("KNOWN_DEAD_IMPORTS", e) for e in KNOWN_DEAD_IMPORTS] + ) + defect_tests = [] + for name, obj in sorted(globals().items()): + if not (name.startswith("test_defect_") and callable(obj)): + continue + marks = [m for m in getattr(obj, "pytestmark", []) if m.name == "xfail"] + assert marks, f"{name} must carry an xfail marker pinning the live defect" + for m in marks: + assert m.kwargs.get("strict") is True, ( + f"{name} carries a NON-STRICT xfail. A non-strict xfail rots silently: " + "it keeps reporting 'expected failure' after the defect is fixed, so the " + "KNOWN_* entry it justifies never gets removed." + ) + reason = m.kwargs.get("reason") or "" + assert len(reason) > 40, f"{name}'s xfail needs a reason naming the defect" + defect_tests.append(name) + + assert len(entries) == len(defect_tests), ( + f"{len(entries)} KNOWN_* suppression(s) but {len(defect_tests)} strict-xfail " + "defect test(s). Every suppression needs one, or it is an unexplained " + f"suppression.\n entries: {sorted(entries)}\n tests: {defect_tests}" + ) + + +def test_guarded_and_first_party_imports_resolve(): + """The highest-value check here. A `from src... import x` inside + `try: ... except Exception: pass` that no longer resolves is invisible forever — + the feature is simply gone, silently.""" + dead = compute_dead_imports(import_sites(), GUARDED_IMPORT_ALLOWLIST) + known = {(f, s) for f, s in KNOWN_DEAD_IMPORTS} + remaining = {(f, s, r) for f, s, r in dead if (f, s) not in known} + assert remaining == set(), "import that does not resolve:\n" + "\n".join( + f" {f}: {s}\n {r}" for f, s, r in sorted(remaining) + ) + + +def test_import_gate_actually_checked_the_lazy_imports(): + """Guard against the gate quietly checking nothing (e.g. an ast change that stops + finding nested imports).""" + sites = import_sites() + guarded = [s for s in sites if s.in_try] + first_party = [s for s in sites if s.module.startswith("src")] + assert len(guarded) > 30, f"only {len(guarded)} try-guarded imports found" + assert len(first_party) > 150, f"only {len(first_party)} first-party imports found" + + +def test_static_link_resolution_coverage_is_reported(): + """We cannot resolve a URL built entirely from a variable. Pin how much we *can* + check so a future change that guts the matcher shows up as a coverage drop.""" + values = link_attr_values() + resolvable = [v for _, v in values if _normalize_link(v)] + assert len(values) > 100, f"only {len(values)} href/action attrs found" + fraction = len(resolvable) / len(values) + assert fraction >= 0.80, ( + f"only {len(resolvable)}/{len(values)} ({fraction:.0%}) of href/action values " + "resolve to a checkable local path — the matcher probably regressed" + ) + + +# --------------------------------------------------------------------------- +# Teeth. The detectors are pure functions of collected data, so we can feed them +# synthetic trees and prove each finds a fresh orphan — without touching a repo file. +# --------------------------------------------------------------------------- + + +def test_teeth_orphan_template_detector_catches_a_new_orphan(): + names = ("base.html", "landing.html", "admin/_partial.html", "ghost/leftover.html") + refs = {"landing.html": frozenset({"base.html", "admin/_partial.html"})} + reachable = compute_reachable_templates(frozenset({"landing.html"}), refs) + assert compute_orphan_templates(names, reachable) == {"ghost/leftover.html"} + + +def test_teeth_orphan_detector_is_transitive_not_just_one_hop(): + """A template reachable only through two includes must not be flagged, and one + reachable only *from* an orphan must be.""" + names = ("a.html", "b.html", "c.html", "x.html", "y.html") + refs = { + "a.html": frozenset({"b.html"}), + "b.html": frozenset({"c.html"}), + "x.html": frozenset({"y.html"}), # x is an orphan, so y is unreachable too + } + reachable = compute_reachable_templates(frozenset({"a.html"}), refs) + assert compute_orphan_templates(names, reachable) == {"x.html", "y.html"} + + +def test_teeth_broken_link_detector_catches_a_form_pointing_at_nothing(): + routes = (Route("POST", "/profile/save", "profile_save"),) + links = ( + Link("profile/edit.html", "POST", "/profile/save", "/profile/save"), + Link("profile/edit.html", "POST", "/profile/vanished", "/profile/vanished"), + ) + broken = compute_broken_links(links, routes, frozenset({"profile/edit.html"})) + assert broken == {("profile/edit.html", "POST", "/profile/vanished")} + + +def test_teeth_broken_link_detector_catches_a_method_mismatch(): + """A form POSTing to a GET-only route 405s; path equality alone would miss it.""" + routes = (Route("GET", "/admin/discussions", "admin_discussions"),) + links = (Link("admin/discussions.html", "POST", "/admin/discussions", "/admin/discussions"),) + broken = compute_broken_links(links, routes, frozenset({"admin/discussions.html"})) + assert broken == {("admin/discussions.html", "POST", "/admin/discussions")} + + +def test_teeth_unreachable_route_detector_catches_a_new_orphan_route(): + routes = ( + Route("GET", "/agent/{agent_id}/dashboard", "agent_dashboard"), + Route("GET", "/agent/{agent_id}/ghost", "agent_ghost"), + ) + links = ( + Link( + "agent/listing.html", + "GET", + "/agent/{{ a.agent_id }}/dashboard", + f"/agent/{HOLE}/dashboard", + ), + ) + unreachable = compute_unreachable_routes( + routes, links, frozenset({"agent/listing.html"}), src_paths=frozenset(), allowlist={} + ) + assert unreachable == {("GET", "/agent/{agent_id}/ghost")} + + +def test_teeth_a_link_inside_an_orphaned_template_does_not_launder_a_route(): + """The exact shape of the defect this gate was built for: the only caller of a route + sits in a template nothing renders. A non-transitive gate would call the route + reachable. Kept synthetic on purpose — the real instance (add_texts.html's Skip + button, the sole caller of POST /onboarding/complete) has since been deleted, and + this is what would catch the next one.""" + routes = (Route("POST", "/onboarding/complete", "complete_onboarding"),) + links = ( + Link( + "onboarding/add_texts.html", "POST", "/onboarding/complete", "/onboarding/complete" + ), + ) + reachable: frozenset[str] = frozenset() # add_texts.html is rendered by nothing + unreachable = compute_unreachable_routes( + routes, links, reachable, src_paths=frozenset(), allowlist={} + ) + assert unreachable == {("POST", "/onboarding/complete")} + + +def test_teeth_dead_import_detector_catches_a_vanished_symbol(): + live = ImportSite( + "src/fake.py", 1, "from src.main import create_app", "src.main", ("create_app",), True + ) + dead = ImportSite( + "src/fake.py", + 2, + "from src.main import _never_existed", + "src.main", + ("_never_existed",), + True, + ) + missing_module = ImportSite( + "src/fake.py", 3, "from src.no_such_mod import x", "src.no_such_mod", ("x",), True + ) + found = compute_dead_imports((live, dead, missing_module), {}) + assert {(f, s) for f, s, _ in found} == { + ("src/fake.py", "from src.main import _never_existed"), + ("src/fake.py", "from src.no_such_mod import x"), + } + + +def test_teeth_path_param_routes_match_by_pattern_not_string_equality(): + assert _link_can_reach(f"/agent/{HOLE}/dashboard", "/agent/{agent_id}/dashboard") + assert _link_credits(f"/agent/{HOLE}/dashboard", "/agent/{agent_id}/dashboard") + # A literal in the link must still match a literal in the route. + assert not _link_credits("/agent/smith/ghost", "/agent/{agent_id}/dashboard") + # Strict matcher must not credit a sibling literal route via a Jinja expression: + # /admin/cohorts/{{ c.id }} does not address /admin/cohorts/topology. + assert not _link_credits(f"/admin/cohorts/{HOLE}", "/admin/cohorts/topology") + # ...but the permissive matcher tolerates it, so we never cry "broken link". + assert _link_can_reach(f"/admin/cohorts/{HOLE}", "/admin/cohorts/topology") + # Segment count must match. + assert not _link_can_reach("/agent/smith", "/agent/{agent_id}/dashboard") + + +def test_teeth_stale_allowlist_detector_catches_both_ways_an_entry_goes_stale(): + routes = ( + Route("GET", "/kept", "kept"), + Route("GET", "/now-linked", "now_linked"), + ) + links = (Link("page.html", "GET", "/now-linked", "/now-linked"),) + allowlist = { + ("GET", "/kept"): "still unreferenced — legitimately entered by hand", + ("GET", "/now-linked"): "was unreferenced when this was written", + ("GET", "/deleted-route"): "route has since been removed", + } + stale = compute_stale_allowlist_entries( + routes, links, frozenset({"page.html"}), frozenset(), allowlist + ) + assert set(stale) == {("GET", "/now-linked"), ("GET", "/deleted-route")} + assert "now referenced" in stale[("GET", "/now-linked")] + assert "no longer exists" in stale[("GET", "/deleted-route")] + + +def test_teeth_a_route_does_not_credit_itself_via_its_own_decorator(): + """Routers mounted without a prefix declare their full path in the decorator. If + that literal counted as a reference, an orphan in public.py/auth.py/invite.py would + be permanently invisible — half the app.""" + source = ( + '"""Module docstring mentioning /docstring-only."""\n' + "app.include_router(r, prefix='/prefix-only')\n" + "@router.get('/decorator-only')\n" + "async def handler():\n" + ' """Docstring mentioning /nested-docstring-only."""\n' + " return RedirectResponse(url='/a-real-redirect')\n" + ) + tree = ast.parse(source) + excluded = _excluded_string_nodes(tree) + kept = { + node.value + for node in ast.walk(tree) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and id(node) not in excluded + } + assert "/a-real-redirect" in kept + for suppressed in ("/decorator-only", "/prefix-only"): + assert suppressed not in kept, f"{suppressed} would credit its own route" + assert not any("docstring-only" in k for k in kept), "docstrings are not callers" + + +def test_teeth_normalizer_handles_the_jinja_shapes_in_this_repo(): + # url-with-query + assert _normalize_link("/admin/discussions?run_id={{ x }}") == ["/admin/discussions"] + # branching value yields both arms + assert _normalize_link( + "{% if r %}/agent/{{ r.agent_id }}/profile/edit{% else %}/agent{% endif %}" + ) == [f"/agent/{HOLE}/profile/edit", "/agent"] + # unresolvable: whole URL comes from a variable + assert _normalize_link("{{ slack_invite_url }}") == [] + # external, anchor, same-page query, static asset + assert _normalize_link("https://orcid.org/{{ u.orcid }}") == [] + assert _normalize_link("#top") == [] + assert _normalize_link("?page={{ page + 1 }}") == [] + assert _normalize_link("/static/js/markdown.js") == [] diff --git a/tests/unit/test_remediate_duplicates.py b/tests/unit/test_remediate_duplicates.py new file mode 100644 index 0000000..d624151 --- /dev/null +++ b/tests/unit/test_remediate_duplicates.py @@ -0,0 +1,839 @@ +"""Pure-logic tests for scripts/migrate/remediate_duplicates.py. + +No database: everything here exercises group classification, strategy selection +and replacement-id generation, which is where a mistake would silently delete a +message or fabricate a Slack timestamp. The DB layer is covered separately +against throwaway Postgres databases (see the report for that change). + +The tool is a script, not an importable package, so it is loaded by path. The +``sys.modules`` registration before ``exec_module`` is load-bearing rather than +tidiness: ``@dataclass`` looks its own module up in ``sys.modules`` to resolve +annotations, and without the entry every dataclass in the file raises +``AttributeError: 'NoneType' object has no attribute '__dict__'`` at import. +""" + +import importlib.util +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from src.agent import ids as ids_mod + +_TOOL_PATH = Path(__file__).resolve().parents[2] / "scripts" / "migrate" / "remediate_duplicates.py" + + +def _load_tool(): + spec = importlib.util.spec_from_file_location("_remediate_duplicates", _TOOL_PATH) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +rd = _load_tool() + + +# --------------------------------------------------------------------------- # +# Fixtures / helpers +# --------------------------------------------------------------------------- # + +BASE_TIME = datetime(2026, 7, 1, tzinfo=UTC) + +#: The 0018 column set, which is what production has when this tool matters most. +COLUMNS_0018 = { + "agent_id": "chen", + "channel_id": "C0SLACK1", + "channel_name": "general", + "message_length": 100, + "phase": "new_post", + "thread_ts": None, + "visibility": "public", +} + +#: The columns 0019 adds. +COLUMNS_0019_EXTRA = { + "content": "body", + "sender_name": "ChenBot", + "is_bot": True, + "posted_at": 1755000001.000017, + "slack_ts": None, + "slack_channel_id": None, + "slack_thread_ts": None, +} + + +def make_row(row_id="a1", ts="1755000001.000017", *, run="run-1", seconds=0, + at_0019=False, **overrides): + """Build a MessageRow. ``overrides`` set individual columns.""" + columns = dict(COLUMNS_0018) + if at_0019: + columns.update(COLUMNS_0019_EXTRA) + columns.update(overrides) + created = BASE_TIME + timedelta(seconds=seconds) + columns.update({"id": row_id, "simulation_run_id": run, "message_ts": ts, + "created_at": created}) + return rd.MessageRow(row_id=row_id, run_id=run, message_ts=ts, created_at=created, + columns=columns) + + +WRITER_SLOTS = {0: "WRITER_ENGINE", 1: "WRITER_WEB", 2: "WRITER_GRANTBOT", + 3: "WRITER_ENGINE_AUX"} + + +def classify(row, *, at_0019=False): + return rd.classify_origin(row, has_slack_columns=at_0019, writer_slots=WRITER_SLOTS, + modulus=100) + + +def group_of(*rows, **kwargs): + return rd.DuplicateGroup(rows[0].run_id, rows[0].message_ts, list(rows), **kwargs) + + +def plan(group, strategy=rd.STRATEGY_RENUMBER, *, used=None, now_us=1_800_000_000_000_000, + at_0019=False): + if used is None: + used = {r.message_ts for r in group.rows} + rd.plan_group(group, strategy=strategy, used=used, now_us=now_us, + has_slack_columns=at_0019, writer_slots=WRITER_SLOTS, modulus=100) + return group + + +# --------------------------------------------------------------------------- # +# ts parsing / formatting +# --------------------------------------------------------------------------- # + +class TestTsShape: + @pytest.mark.parametrize( + ("ts", "expected"), + [ + ("1755000001.000017", 1755000001000017), + ("0.000000", 0), + ("1755000005.000000", 1755000005000000), + ("1755000005.999999", 1755000005999999), + ], + ) + def test_parses_ts_shaped_ids(self, ts, expected): + assert rd.parse_ts_us(ts) == expected + + @pytest.mark.parametrize( + "ts", + [ + None, "", "not-a-slack-ts", "1755000005", "1755000005.", ".000001", + # Fewer than six fractional digits is AMBIGUOUS (2µs or 200000µs?), so + # it is refused rather than guessed at. + "1755000005.2", "1755000005.00001", + "1755000005.0000001", # seven digits + "1755000005.00000a", + "-1755000005.000001", + " 1755000005.000001", + "1755000005.000001 ", + "1755000005.000001\n", + ], + ) + def test_refuses_everything_else(self, ts): + assert rd.parse_ts_us(ts) is None + + def test_format_round_trips(self): + for us in (0, 1, 999_999, 1_000_000, 1755000005000099): + assert rd.parse_ts_us(rd.format_us(us)) == us + + def test_format_agrees_with_the_minter_it_must_match(self): + # If these two ever disagree, remediated ids stop being the same shape as + # minted ones and float(ts) ordering silently changes meaning. + for us in (0, 1, 999_999, 1_000_000, 1755000005000099, 1785894150064299): + assert rd.format_us(us) == ids_mod._fmt(us) + + +# --------------------------------------------------------------------------- # +# Origin classification +# --------------------------------------------------------------------------- # + +class TestClassifyOrigin: + def test_local_channel_is_confirmed_local(self): + origin = classify(make_row(channel_id="local:cryo-em")) + assert origin.verdict == rd.ORIGIN_LOCAL_CONFIRMED + assert "never" in origin.evidence + + def test_malformed_ts_is_confirmed_local(self): + origin = classify(make_row(ts="not-a-slack-ts")) + assert origin.verdict == rd.ORIGIN_LOCAL_CONFIRMED + assert "not ts-shaped" in origin.evidence + + def test_slack_ts_equal_to_message_ts_is_confirmed_slack(self): + row = make_row(ts="1755000001.000017", at_0019=True, slack_ts="1755000001.000017") + assert classify(row, at_0019=True).verdict == rd.ORIGIN_SLACK_CONFIRMED + + def test_slack_ts_differing_from_message_ts_is_confirmed_local(self): + # 0019 keeps the two apart, so message_ts is a canonical id by construction. + row = make_row(ts="1755000001.000017", at_0019=True, slack_ts="1755000009.123456") + assert classify(row, at_0019=True).verdict == rd.ORIGIN_LOCAL_CONFIRMED + + def test_slack_ts_columns_ignored_when_the_revision_predates_them(self): + # Same row, but told the schema has no slack_ts: must fall through to the + # residue heuristic rather than trusting a column it cannot see. + row = make_row(ts="1755000001.000017", at_0019=True, slack_ts="1755000001.000017") + assert classify(row, at_0019=False).verdict == rd.ORIGIN_SLACK_PRESUMED + + @pytest.mark.parametrize( + ("residue", "writer"), + [("000000", "WRITER_ENGINE"), ("000001", "WRITER_WEB"), + ("000002", "WRITER_GRANTBOT"), ("000003", "WRITER_ENGINE_AUX"), + ("123400", "WRITER_ENGINE"), ("999903", "WRITER_ENGINE_AUX")], + ) + def test_writer_slot_residue_is_presumed_local(self, residue, writer): + origin = classify(make_row(ts=f"1755000007.{residue}")) + assert origin.verdict == rd.ORIGIN_LOCAL_PRESUMED + assert writer in origin.evidence + + @pytest.mark.parametrize("residue", ["000017", "000042", "000057", "000088", "000091", + "123456", "999999"]) + def test_anything_else_in_a_slack_channel_is_presumed_slack(self, residue): + origin = classify(make_row(ts=f"1755000007.{residue}")) + assert origin.verdict == rd.ORIGIN_SLACK_PRESUMED + assert origin.is_slack + + def test_local_channel_beats_the_residue_heuristic(self): + # Legacy rows minted by the pre-writer-slot f"{time.time():.6f}" scheme have + # arbitrary residues; the channel is the stronger signal and must win. + origin = classify(make_row(ts="1755000003.000057", channel_id="local:cryo-em")) + assert origin.verdict == rd.ORIGIN_LOCAL_CONFIRMED + + def test_is_slack_property_covers_exactly_the_two_slack_verdicts(self): + assert rd.Origin(rd.ORIGIN_SLACK_CONFIRMED, "").is_slack + assert rd.Origin(rd.ORIGIN_SLACK_PRESUMED, "").is_slack + assert not rd.Origin(rd.ORIGIN_LOCAL_CONFIRMED, "").is_slack + assert not rd.Origin(rd.ORIGIN_LOCAL_PRESUMED, "").is_slack + + +# --------------------------------------------------------------------------- # +# Renumber safety +# --------------------------------------------------------------------------- # + +class TestRenumberVerdict: + def test_confirmed_local_is_safe(self): + verdict, _ = rd.renumber_verdict( + make_row(), rd.Origin(rd.ORIGIN_LOCAL_CONFIRMED, ""), has_slack_columns=False + ) + assert verdict == rd.RENUMBER_SAFE + + def test_presumed_local_is_safe_but_says_what_it_costs_if_wrong(self): + verdict, reason = rd.renumber_verdict( + make_row(), rd.Origin(rd.ORIGIN_LOCAL_PRESUMED, ""), has_slack_columns=False + ) + assert verdict == rd.RENUMBER_SAFE + assert "backfill_slack_ts" in reason + + def test_confirmed_slack_is_safe_at_0019_because_slack_ts_holds_the_timestamp(self): + verdict, reason = rd.renumber_verdict( + make_row(), rd.Origin(rd.ORIGIN_SLACK_CONFIRMED, ""), has_slack_columns=True + ) + assert verdict == rd.RENUMBER_SAFE + assert "slack_ts keeps" in reason + + def test_presumed_slack_is_never_safe(self): + for has_slack in (False, True): + verdict, reason = rd.renumber_verdict( + make_row(), rd.Origin(rd.ORIGIN_SLACK_PRESUMED, ""), + has_slack_columns=has_slack, + ) + assert verdict == rd.RENUMBER_UNSAFE + assert "only record" in reason + + def test_the_0018_reason_names_the_missing_column(self): + _, reason = rd.renumber_verdict( + make_row(), rd.Origin(rd.ORIGIN_SLACK_PRESUMED, ""), has_slack_columns=False + ) + assert "0018 schema has no slack_ts column" in reason + + +# --------------------------------------------------------------------------- # +# Payload comparison: the whole basis for "safe to delete" +# --------------------------------------------------------------------------- # + +class TestPayloadIdentity: + def test_rows_differing_only_in_id_are_identical(self): + assert make_row("a1").payload() == make_row("a2").payload() + + def test_rows_differing_only_in_created_at_are_identical(self): + assert make_row("a1", seconds=0).payload() == make_row("a2", seconds=5).payload() + + @pytest.mark.parametrize( + ("column", "value"), + [ + ("agent_id", "patel"), + ("channel_id", "C0SLACK2"), + ("channel_name", "proteomics"), + ("message_length", 0), + ("phase", "thread_reply"), + ("thread_ts", "1755000000.000005"), + ("visibility", "collab_private"), + ], + ) + def test_any_other_column_makes_them_divergent_at_0018(self, column, value): + assert make_row("a1").payload() != make_row("a2", **{column: value}).payload() + + @pytest.mark.parametrize( + ("column", "value"), + [ + ("content", "something else entirely"), + ("content", ""), + ("sender_name", "PatelBot"), + ("is_bot", False), + ("posted_at", 1.0), + ("slack_ts", "1755000001.000017"), + ("slack_channel_id", "C0SLACK9"), + ("slack_thread_ts", "1755000000.000005"), + ], + ) + def test_the_0019_columns_count_as_payload_too(self, column, value): + left = make_row("a1", at_0019=True) + right = make_row("a2", at_0019=True, **{column: value}) + assert left.payload() != right.payload() + + @pytest.mark.parametrize( + ("channel", "expected_resolution"), + [ + # Renumberable rows: the empty twin moves, both survive. + ("local:j", rd.RESOLUTION_RENUMBER), + # Slack-presumed rows: no id may move either, so it goes to a human. + ("C0SLACK1", rd.RESOLUTION_NEEDS_HUMAN), + ], + ) + @pytest.mark.parametrize("strategy", rd.STRATEGIES) + def test_content_only_divergence_is_never_resolved_by_deleting( + self, channel, expected_resolution, strategy + ): + # The headline guarantee: a row carrying text its twin does not is NOT a + # redundant copy, and no strategy may drop it -- whatever else matches. + left = make_row("a1", at_0019=True, content="IMPORTANT", channel_id=channel) + right = make_row("a2", at_0019=True, content="", channel_id=channel, seconds=1) + group = plan(group_of(left, right), strategy, at_0019=True) + assert group.kind == rd.KIND_DIVERGENT + assert group.resolution == expected_resolution + assert rd.ACTION_DELETE not in [r.action for r in group.rows] + + def test_sort_key_survives_a_null_created_at(self): + row = make_row("a1") + row.created_at = None + assert row.sort_key()[0] is False # sorts first, does not raise + + +# --------------------------------------------------------------------------- # +# Replacement id generation +# --------------------------------------------------------------------------- # + +class TestMintReplacementTs: + def test_lands_in_the_remediation_slot(self): + new = rd.mint_replacement_ts("1755000005.000000", {"1755000005.000000"}, now_us=1) + assert rd.parse_ts_us(new) % 100 == rd.REMEDIATION_WRITER_SLOT + assert new == "1755000005.000099" + + def test_never_lands_in_a_live_writer_slot(self): + for residue in range(100): + new = rd.mint_replacement_ts( + f"1755000005.{residue:06d}", {f"1755000005.{residue:06d}"}, now_us=1 + ) + assert rd.parse_ts_us(new) % 100 not in WRITER_SLOTS + + def test_stays_in_the_original_neighbourhood_so_ordering_holds(self): + original = "1755000005.000000" + new = rd.mint_replacement_ts(original, {original}, now_us=1) + delta = rd.parse_ts_us(new) - rd.parse_ts_us(original) + assert 0 < delta < 100 + + def test_is_always_after_the_id_it_replaces(self): + for residue in range(100): + original = f"1755000005.{residue:06d}" + new = rd.mint_replacement_ts(original, {original}, now_us=1) + assert rd.parse_ts_us(new) > rd.parse_ts_us(original) + + def test_probes_past_taken_ids(self): + original = "1755000005.000000" + used = {original, "1755000005.000099", "1755000005.000199", "1755000005.000299"} + assert rd.mint_replacement_ts(original, used, now_us=1) == "1755000005.000399" + + def test_probes_past_an_id_that_is_only_a_thread_pointer(self): + # load_used_ids feeds thread_ts and thread_decisions.thread_id in here too, + # so a replacement can never quietly re-parent someone else's reply. + original = "1755000005.000000" + used = {original, "1755000005.000099"} + assert rd.mint_replacement_ts(original, used, now_us=1) == "1755000005.000199" + + def test_an_original_already_in_the_remediation_slot_still_moves(self): + original = "1755000005.000099" + assert rd.mint_replacement_ts(original, {original}, now_us=1) == "1755000005.000199" + + def test_an_original_in_the_remediation_slot_moves_even_if_used_is_empty(self): + original = "1755000005.000099" + assert rd.mint_replacement_ts(original, set(), now_us=1) != original + + def test_malformed_original_falls_back_to_the_clock(self): + new = rd.mint_replacement_ts("not-a-slack-ts", set(), now_us=1_800_000_000_000_000) + assert new == "1800000000.000099" + + def test_gives_up_rather_than_looping_forever(self): + original = "1755000005.000000" + used = {original} | { + rd.format_us((rd.parse_ts_us(original) // 100 + n) * 100 + 99) for n in range(20) + } + with pytest.raises(RuntimeError, match="no free id"): + rd.mint_replacement_ts(original, used, now_us=1, max_probes=20) + + def test_a_run_of_replacements_is_collision_free(self): + used = {"1755000005.000000"} + minted = [] + for _ in range(50): + new = rd.mint_replacement_ts("1755000005.000000", used, now_us=1) + used.add(new) + minted.append(new) + assert len(set(minted)) == 50 + assert minted == sorted(minted, key=rd.parse_ts_us) + + +# --------------------------------------------------------------------------- # +# Anchor selection +# --------------------------------------------------------------------------- # + +class TestPickAnchor: + def _rows(self): + rows = [make_row("a1", seconds=0), make_row("a2", seconds=5), + make_row("a3", seconds=9)] + for row in rows: + row.renumber_verdict = rd.RENUMBER_SAFE + return rows + + def test_keep_earliest_takes_the_oldest(self): + assert rd.pick_anchor(self._rows(), strategy=rd.STRATEGY_KEEP_EARLIEST, + reply_channel_ids=set()).row_id == "a1" + + def test_keep_latest_takes_the_newest(self): + assert rd.pick_anchor(self._rows(), strategy=rd.STRATEGY_KEEP_LATEST, + reply_channel_ids=set()).row_id == "a3" + + def test_the_unsafe_row_always_wins(self): + rows = self._rows() + rows[2].renumber_verdict = rd.RENUMBER_UNSAFE + for strategy in rd.STRATEGIES: + assert rd.pick_anchor(rows, strategy=strategy, + reply_channel_ids=set()).row_id == "a3" + + def test_a_thread_root_in_the_replies_channel_beats_the_clock(self): + rows = self._rows() + rows[1].columns["channel_id"] = "C0THREAD" + assert rd.pick_anchor(rows, strategy=rd.STRATEGY_KEEP_EARLIEST, + reply_channel_ids={"C0THREAD"}).row_id == "a2" + + def test_an_unsafe_row_still_beats_the_replies_channel(self): + rows = self._rows() + rows[1].columns["channel_id"] = "C0THREAD" + rows[0].renumber_verdict = rd.RENUMBER_UNSAFE + assert rd.pick_anchor(rows, strategy=rd.STRATEGY_KEEP_EARLIEST, + reply_channel_ids={"C0THREAD"}).row_id == "a1" + + def test_ties_break_on_the_primary_key_so_runs_are_reproducible(self): + rows = [make_row("a9", seconds=0), make_row("a2", seconds=0)] + for row in rows: + row.renumber_verdict = rd.RENUMBER_SAFE + assert rd.pick_anchor(rows, strategy=rd.STRATEGY_KEEP_EARLIEST, + reply_channel_ids=set()).row_id == "a2" + + +# --------------------------------------------------------------------------- # +# Group classification and strategy selection +# --------------------------------------------------------------------------- # + +class TestPlanGroupRedundant: + def _identical_slack_pair(self): + return group_of(make_row("a1", seconds=0), make_row("a2", seconds=5)) + + def test_renumber_refuses_a_slack_born_identical_pair_and_names_the_fix(self): + group = plan(self._identical_slack_pair(), rd.STRATEGY_RENUMBER) + assert group.kind == rd.KIND_REDUNDANT + assert group.resolution == rd.RESOLUTION_NEEDS_DELETE_STRATEGY + assert not group.resolved + assert "--strategy keep-earliest" in group.reason + assert all(r.action == rd.ACTION_KEEP for r in group.rows) + + def test_keep_earliest_deletes_the_later_copies(self): + group = plan(self._identical_slack_pair(), rd.STRATEGY_KEEP_EARLIEST) + assert group.resolution == rd.RESOLUTION_DELETE + assert group.anchor_id == "a1" + assert [r.action for r in group.rows] == [rd.ACTION_KEEP, rd.ACTION_DELETE] + + def test_keep_latest_deletes_the_earlier_copies(self): + group = plan(self._identical_slack_pair(), rd.STRATEGY_KEEP_LATEST) + assert group.resolution == rd.RESOLUTION_DELETE + assert group.anchor_id == "a2" + assert [r.action for r in group.rows] == [rd.ACTION_DELETE, rd.ACTION_KEEP] + + def test_a_three_way_identical_group_keeps_exactly_one(self): + group = group_of(make_row("a1", seconds=0), make_row("a2", seconds=1), + make_row("a3", seconds=2)) + plan(group, rd.STRATEGY_KEEP_EARLIEST) + assert [r.action for r in group.rows].count(rd.ACTION_KEEP) == 1 + assert [r.action for r in group.rows].count(rd.ACTION_DELETE) == 2 + + def test_an_all_local_identical_group_can_be_renumbered_instead(self): + group = group_of(make_row("a1", channel_id="local:x", seconds=0), + make_row("a2", channel_id="local:x", seconds=5)) + plan(group, rd.STRATEGY_RENUMBER) + assert group.kind == rd.KIND_REDUNDANT + assert group.resolution == rd.RESOLUTION_RENUMBER + assert group.resolved + # ...but it says out loud that this doubles the message in rebuilt history. + assert "appear twice" in group.reason + assert group.rows[1].new_message_ts is not None + + def test_at_0019_an_identical_confirmed_slack_pair_is_renumberable(self): + # The same data that 0018 can only fix by deleting: at 0019 slack_ts holds + # the Slack timestamp, so the canonical id is free to move. + rows = [ + make_row("a1", at_0019=True, slack_ts="1755000001.000017", seconds=0), + make_row("a2", at_0019=True, slack_ts="1755000001.000017", seconds=5), + ] + group = plan(group_of(*rows), rd.STRATEGY_RENUMBER, at_0019=True) + assert group.kind == rd.KIND_REDUNDANT + assert group.resolution == rd.RESOLUTION_RENUMBER + + +class TestPlanGroupDivergent: + def test_one_slack_row_plus_one_local_row_renumbers_the_local_one(self): + slack = make_row("a1", message_length=250, seconds=0) + local = make_row("a2", channel_id="local:cryo-em", message_length=0, seconds=1) + group = plan(group_of(slack, local), rd.STRATEGY_RENUMBER) + assert group.kind == rd.KIND_DIVERGENT + assert group.resolution == rd.RESOLUTION_RENUMBER + assert group.anchor_id == "a1" + assert slack.action == rd.ACTION_KEEP + assert local.action == rd.ACTION_RENUMBER + assert local.new_message_ts == "1755000001.000099" + + def test_two_slack_born_divergent_rows_need_a_human(self): + group = plan( + group_of(make_row("a1", message_length=310, seconds=0), + make_row("a2", message_length=44, channel_id="C0SLACK2", seconds=1)), + rd.STRATEGY_RENUMBER, + ) + assert group.resolution == rd.RESOLUTION_NEEDS_HUMAN + assert not group.resolved + assert "REFUSING to guess" in group.reason + assert all(r.action == rd.ACTION_KEEP for r in group.rows) + + @pytest.mark.parametrize("strategy", rd.STRATEGIES) + def test_no_strategy_can_talk_it_into_guessing(self, strategy): + group = plan( + group_of(make_row("a1", message_length=310, seconds=0), + make_row("a2", message_length=44, channel_id="C0SLACK2", seconds=1)), + strategy, + ) + assert group.resolution == rd.RESOLUTION_NEEDS_HUMAN + + @pytest.mark.parametrize("strategy", rd.STRATEGIES) + def test_a_divergent_group_is_never_resolved_by_deletion(self, strategy): + group = plan( + group_of(make_row("a1", channel_id="local:x", message_length=1, seconds=0), + make_row("a2", channel_id="local:x", message_length=2, seconds=1)), + strategy, + ) + assert group.resolution == rd.RESOLUTION_RENUMBER + assert rd.ACTION_DELETE not in [r.action for r in group.rows] + + def test_a_three_way_divergent_group_gets_two_distinct_replacements(self): + rows = [make_row(f"a{n}", channel_id="local:x", message_length=n, seconds=n) + for n in (1, 2, 3)] + group = plan(group_of(*rows), rd.STRATEGY_RENUMBER) + assert group.resolution == rd.RESOLUTION_RENUMBER + new = [r.new_message_ts for r in rows if r.new_message_ts] + assert new == ["1755000001.000099", "1755000001.000199"] + assert len(set(new)) == 2 + + def test_the_reason_names_the_row_whose_ts_is_frozen(self): + slack = make_row("a1", message_length=250, seconds=0) + local = make_row("a2", channel_id="local:x", message_length=0, seconds=1) + group = plan(group_of(slack, local), rd.STRATEGY_RENUMBER) + assert "a1" in group.reason + + def test_replacements_avoid_ids_already_used_elsewhere_in_the_run(self): + rows = [make_row("a1", channel_id="local:x", message_length=1, seconds=0), + make_row("a2", channel_id="local:x", message_length=2, seconds=1)] + used = {"1755000001.000017", "1755000001.000099", "1755000001.000199"} + plan(group_of(*rows), rd.STRATEGY_RENUMBER, used=used) + assert rows[1].new_message_ts == "1755000001.000299" + + def test_planning_adds_its_own_output_to_the_used_set(self): + used = {"1755000001.000017"} + rows = [make_row("a1", channel_id="local:x", message_length=1, seconds=0), + make_row("a2", channel_id="local:x", message_length=2, seconds=1)] + plan(group_of(*rows), rd.STRATEGY_RENUMBER, used=used) + assert rows[1].new_message_ts in used + + def test_two_groups_sharing_a_used_set_cannot_collide(self): + used = {"1755000001.000017", "1755000001.000018"} + first = group_of(make_row("a1", channel_id="local:x", message_length=1, seconds=0), + make_row("a2", channel_id="local:x", message_length=2, seconds=1)) + second = group_of( + make_row("b1", ts="1755000001.000018", channel_id="local:x", + message_length=3, seconds=2), + make_row("b2", ts="1755000001.000018", channel_id="local:x", + message_length=4, seconds=3), + ) + plan(first, rd.STRATEGY_RENUMBER, used=used) + plan(second, rd.STRATEGY_RENUMBER, used=used) + assert first.rows[1].new_message_ts != second.rows[1].new_message_ts + + def test_a_malformed_ts_group_is_renumbered_from_the_clock(self): + rows = [make_row("a1", ts="not-a-slack-ts", message_length=7, seconds=0), + make_row("a2", ts="not-a-slack-ts", message_length=9, seconds=1)] + group = plan(group_of(*rows), rd.STRATEGY_RENUMBER, now_us=1_800_000_000_000_000) + assert group.resolution == rd.RESOLUTION_RENUMBER + assert rows[1].new_message_ts == "1800000000.000099" + + def test_replanning_a_group_clears_the_previous_plan(self): + # plan_group is called once per group per run, but it must be re-entrant: + # --apply plans under the table lock, and a stale action would be a write. + rows = [make_row("a1", channel_id="local:x", message_length=1, seconds=0), + make_row("a2", channel_id="local:x", message_length=2, seconds=1)] + group = group_of(*rows) + plan(group, rd.STRATEGY_RENUMBER) + assert rows[1].action == rd.ACTION_RENUMBER + # Re-plan the same objects as a NEEDS_HUMAN group. + rows[0].columns["channel_id"] = "C0SLACK1" + rows[1].columns["channel_id"] = "C0SLACK2" + plan(group, rd.STRATEGY_RENUMBER) + assert group.resolution == rd.RESOLUTION_NEEDS_HUMAN + assert all(r.action == rd.ACTION_KEEP for r in rows) + assert all(r.new_message_ts is None for r in rows) + + +class TestThreadAwareness: + def test_a_referenced_group_reports_its_inbound_pointers(self): + group = group_of( + make_row("a1", seconds=0), + make_row("a2", channel_id="local:mirror", message_length=33, seconds=1), + thread_reply_count=2, thread_decision_count=1, + thread_reply_channel_ids={"C0SLACK1"}, + ) + plan(group, rd.STRATEGY_RENUMBER) + assert group.referenced + # The row in the replies' channel keeps the ts, so the pointers still land. + assert group.anchor_id == "a1" + + def test_an_unreferenced_group_is_not_flagged(self): + group = group_of(make_row("a1"), make_row("a2", channel_id="local:x")) + plan(group, rd.STRATEGY_RENUMBER) + assert not group.referenced + + def test_needs_human_advice_mentions_the_thread_pointers(self): + group = group_of(make_row("a1", message_length=1), + make_row("a2", message_length=2, channel_id="C0SLACK2"), + thread_reply_count=2, thread_decision_count=1) + plan(group, rd.STRATEGY_RENUMBER) + advice = "\n".join(rd.needs_human_advice(group)) + assert "conversations.replies" in advice + assert "2 reply row(s)" in advice + assert "thread root" in advice + + def test_needs_human_advice_omits_the_thread_step_when_nothing_points_here(self): + group = group_of(make_row("a1", message_length=1), + make_row("a2", message_length=2, channel_id="C0SLACK2")) + plan(group, rd.STRATEGY_RENUMBER) + advice = "\n".join(rd.needs_human_advice(group)) + assert "thread root" not in advice + assert "conversations.replies" in advice + + +# --------------------------------------------------------------------------- # +# The id-scheme guard +# --------------------------------------------------------------------------- # + +class _FakeIds: + WRITER_SLOT_MODULUS = 100 + WRITER_ENGINE = 0 + WRITER_WEB = 1 + __file__ = "/fake/src/agent/ids.py" + + +class TestLoadIdScheme: + def test_reads_the_real_module(self): + modulus, slots, path = rd.load_id_scheme() + assert modulus == ids_mod.WRITER_SLOT_MODULUS + assert slots[ids_mod.WRITER_ENGINE] == "WRITER_ENGINE" + assert slots[ids_mod.WRITER_WEB] == "WRITER_WEB" + assert path.endswith("src/agent/ids.py") + + def _with_fake(self, monkeypatch, fake): + """Simulate load_id_scheme() importing a stale/altered baked copy of src/. + + Patching ``sys.modules["src.agent.ids"]`` alone does NOT work, and finding + that out is the reason this helper exists: ``from src.agent import ids`` + imports the ``src.agent`` PACKAGE and then does a plain ``getattr`` for + ``ids``. Once the submodule has been imported anywhere, that attribute is + already bound and sys.modules is never consulted, so the fake was ignored + and five of these tests were silently asserting against the real module. + Both are patched below so the substitution holds either way. + """ + import src.agent + + monkeypatch.setattr(src.agent, "ids", fake) + monkeypatch.setitem(sys.modules, "src.agent.ids", fake) + + def test_accepts_a_matching_scheme(self, monkeypatch): + self._with_fake(monkeypatch, _FakeIds) + modulus, slots, path = rd.load_id_scheme() + assert modulus == 100 + assert slots == {0: "WRITER_ENGINE", 1: "WRITER_WEB"} + assert path == "/fake/src/agent/ids.py" + + def test_rejects_a_different_modulus(self, monkeypatch): + class Drifted(_FakeIds): + WRITER_SLOT_MODULUS = 1000 + + self._with_fake(monkeypatch, Drifted) + with pytest.raises(rd.SchemeError, match="WRITER_SLOT_MODULUS"): + rd.load_id_scheme() + + def test_rejects_a_scheme_that_has_claimed_the_remediation_slot(self, monkeypatch): + class Claimed(_FakeIds): + WRITER_SOMETHING_NEW = rd.REMEDIATION_WRITER_SLOT + + self._with_fake(monkeypatch, Claimed) + with pytest.raises(rd.SchemeError, match="is now claimed by"): + rd.load_id_scheme() + + def test_rejects_a_scheme_with_no_writers_at_all(self, monkeypatch): + class Empty: + WRITER_SLOT_MODULUS = 100 + __file__ = "/fake/ids.py" + + self._with_fake(monkeypatch, Empty) + with pytest.raises(rd.SchemeError, match="no WRITER_"): + rd.load_id_scheme() + + def test_ignores_non_integer_writer_attributes(self, monkeypatch): + class Mixed(_FakeIds): + WRITER_NAMES = ("engine", "web") + WRITER_ENABLED = True # bool is an int subclass; must not become a slot + + self._with_fake(monkeypatch, Mixed) + _, slots, _ = rd.load_id_scheme() + assert slots == {0: "WRITER_ENGINE", 1: "WRITER_WEB"} + + def test_the_remediation_slot_is_in_range_and_free_in_the_real_scheme(self): + _, slots, _ = rd.load_id_scheme() + assert 0 <= rd.REMEDIATION_WRITER_SLOT < ids_mod.WRITER_SLOT_MODULUS + assert rd.REMEDIATION_WRITER_SLOT not in slots + + +# --------------------------------------------------------------------------- # +# DSN handling and the report envelope +# --------------------------------------------------------------------------- # + +class TestDsn: + @pytest.mark.parametrize( + ("given", "expected"), + [ + ("postgresql+asyncpg://u:p@h:5432/d", "postgresql+asyncpg://u:p@h:5432/d"), + ("postgresql://u:p@h:5432/d", "postgresql+asyncpg://u:p@h:5432/d"), + ("postgres://u:p@h:5432/d", "postgresql+asyncpg://u:p@h:5432/d"), + ("postgresql+psycopg://u:p@h/d", "postgresql+psycopg://u:p@h/d"), + ], + ) + def test_normalise(self, given, expected): + assert rd.normalise_dsn(given) == expected + + def test_redaction_hides_the_password_and_keeps_the_host(self): + masked = rd.redact_dsn("postgresql+asyncpg://copi:s3cret@db.example:5432/copi") + assert "s3cret" not in masked + assert "db.example:5432/copi" in masked + assert "copi:***@" in masked + + def test_redaction_leaves_a_passwordless_dsn_alone(self): + assert rd.redact_dsn("postgresql+asyncpg://h/d") == "postgresql+asyncpg://h/d" + + +class TestJsonEnvelope: + def test_group_json_is_serialisable_and_complete(self): + import json + + group = plan( + group_of(make_row("a1", at_0019=True, message_length=250, seconds=0), + make_row("a2", at_0019=True, channel_id="local:x", seconds=1)), + rd.STRATEGY_RENUMBER, at_0019=True, + ) + blob = json.loads(json.dumps(rd.group_to_json(group))) + assert blob["kind"] == rd.KIND_DIVERGENT + assert blob["resolution"] == rd.RESOLUTION_RENUMBER + assert blob["row_count"] == 2 + assert blob["resolved"] is True + assert {r["action"] for r in blob["rows"]} == {rd.ACTION_KEEP, rd.ACTION_RENUMBER} + # created_at is a datetime in the row; it has to survive json.dumps. + assert isinstance(blob["rows"][0]["columns"]["created_at"], str) + assert blob["rows"][0]["origin_evidence"] + + def test_envelope_reports_the_counts_and_the_exit_code(self): + schema = {"alembic_revision": "0018", "has_content": False, "has_slack_ts": False, + "constraint_present": False, "total_rows": 127, + "null_message_ts_rows": 65} + resolvable = plan( + group_of(make_row("a1", message_length=250, seconds=0), + make_row("a2", channel_id="local:x", seconds=1)), + rd.STRATEGY_RENUMBER, + ) + refused = plan( + group_of(make_row("b1", ts="1755000002.000042", message_length=1, seconds=0), + make_row("b2", ts="1755000002.000042", message_length=2, + channel_id="C0SLACK2", seconds=1)), + rd.STRATEGY_RENUMBER, + ) + envelope = rd._envelope( + schema, "postgresql+asyncpg://u:p@h/d", rd.STRATEGY_RENUMBER, False, + [resolvable, refused], None, rd.EXIT_REMAIN, + ) + assert envelope["summary"]["duplicate_groups"] == 2 + assert envelope["summary"]["rows_in_groups"] == 4 + assert envelope["summary"]["rows_to_renumber"] == 1 + assert envelope["summary"]["rows_to_delete"] == 0 + assert envelope["summary"]["unresolved_groups"] == 1 + assert envelope["summary"]["by_resolution"] == { + rd.RESOLUTION_RENUMBER: 1, rd.RESOLUTION_NEEDS_HUMAN: 1, + } + assert envelope["exit_code"] == rd.EXIT_REMAIN + assert "p@" not in envelope["database"] + + def test_truncation_keeps_the_length_visible(self): + rendered = rd._fmt_value("x" * 500) + assert "500 chars" in rendered + assert len(rendered) < 120 + + +class TestExitCodes: + def test_the_contract_is_the_documented_one(self): + assert (rd.EXIT_CLEAN, rd.EXIT_REMAIN, rd.EXIT_FOUND_DRY_RUN) == (0, 1, 2) + + def test_operational_and_usage_codes_cannot_be_mistaken_for_a_verdict(self): + assert rd.EXIT_OPERATIONAL not in (0, 1, 2) + assert rd.EXIT_USAGE not in (0, 1, 2, rd.EXIT_OPERATIONAL) + + def test_a_usage_error_exits_64_not_argparse_default_2(self): + parser = rd.build_parser() + with pytest.raises(SystemExit) as exc: + parser.parse_args(["--strategy", "not-a-strategy"]) + assert exc.value.code == rd.EXIT_USAGE + + def test_the_default_strategy_is_the_non_destructive_one(self): + args = rd.build_parser().parse_args([]) + assert args.strategy == rd.STRATEGY_RENUMBER + assert args.apply is False + assert args.as_json is False + + def test_main_reports_a_missing_dsn_as_operational(self, monkeypatch): + monkeypatch.delenv("DATABASE_URL", raising=False) + assert rd.main([]) == rd.EXIT_OPERATIONAL + + def test_unresolved_resolutions_are_exactly_the_two_refusals(self): + assert rd.UNRESOLVED == { + rd.RESOLUTION_NEEDS_DELETE_STRATEGY, rd.RESOLUTION_NEEDS_HUMAN, + } diff --git a/tests/unit/test_roles.py b/tests/unit/test_roles.py new file mode 100644 index 0000000..c9fa9a8 --- /dev/null +++ b/tests/unit/test_roles.py @@ -0,0 +1,166 @@ +import logging + +from src.agent import roles +from src.agent.roles import DEFAULT_TOOLS, RoleSpec, load_role + + +def _write_role(tmp_path, monkeypatch, name, toml_text): + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + d = tmp_path / "roles" / name + d.mkdir(parents=True) + (d / "role.toml").write_text(toml_text, encoding="utf-8") + + +def test_resolve_falls_back_to_global_when_no_role_override(tmp_path, monkeypatch): + monkeypatch.setattr(roles, "PROMPTS_DIR", tmp_path) + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + (tmp_path / "agent-system.md").write_text("GLOBAL", encoding="utf-8") + + p = roles.resolve_prompt_path("scout_hub", "agent-system.md") + + assert p == tmp_path / "agent-system.md" + assert p.read_text(encoding="utf-8") == "GLOBAL" + + +def test_resolve_prefers_role_override_when_present(tmp_path, monkeypatch): + monkeypatch.setattr(roles, "PROMPTS_DIR", tmp_path) + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + (tmp_path / "agent-system.md").write_text("GLOBAL", encoding="utf-8") + role_dir = tmp_path / "roles" / "scout_hub" + role_dir.mkdir(parents=True) + (role_dir / "agent-system.md").write_text("HUB", encoding="utf-8") + + p = roles.resolve_prompt_path("scout_hub", "agent-system.md") + + assert p == role_dir / "agent-system.md" + assert p.read_text(encoding="utf-8") == "HUB" + + +def test_pi_lab_resolves_to_global_even_if_role_dir_absent(tmp_path, monkeypatch): + monkeypatch.setattr(roles, "PROMPTS_DIR", tmp_path) + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + (tmp_path / "phase5-new-post.md").write_text("DEFAULT", encoding="utf-8") + + p = roles.resolve_prompt_path("pi_lab", "phase5-new-post.md") + + assert p == tmp_path / "phase5-new-post.md" + + +def test_missing_manifest_yields_defaults(tmp_path, monkeypatch): + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + spec = load_role("pi_lab") + assert spec == RoleSpec(name="pi_lab", label="pi_lab", tools=DEFAULT_TOOLS) + + +def test_manifest_sets_label_and_tool_allow_list(tmp_path, monkeypatch): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\n' + 'tools = ["retrieve_profile", "search_prior_art"]\n', + ) + # search_prior_art must exist in TOOL_DEFINITIONS by the time this runs + # (Task 7). Until then this asserts only the known tool survives. + spec = load_role("scout_hub") + assert spec.name == "scout_hub" + assert spec.label == "Scout Hub" + assert "retrieve_profile" in spec.tools + + +def test_unknown_tool_is_dropped_and_logged(tmp_path, monkeypatch, caplog): + _write_role( + tmp_path, monkeypatch, "weird", + 'tools = ["retrieve_profile", "does_not_exist"]\n', + ) + with caplog.at_level(logging.WARNING): + spec = load_role("weird") + assert "does_not_exist" not in spec.tools + assert "retrieve_profile" in spec.tools + assert any("does_not_exist" in r.message for r in caplog.records) + + +def test_malformed_toml_falls_back_to_defaults(tmp_path, monkeypatch, caplog): + _write_role(tmp_path, monkeypatch, "broken", "tools = [not valid toml") + with caplog.at_level(logging.ERROR): + spec = load_role("broken") + assert spec.tools == DEFAULT_TOOLS + assert spec.label == "broken" +def test_role_rate_override_is_read_when_positive(tmp_path, monkeypatch): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\ncalls_per_load_per_window = 20\n', + ) + assert load_role("scout_hub").calls_per_load_per_window == 20 + + +def test_role_rate_override_defaults_to_none(tmp_path, monkeypatch): + _write_role(tmp_path, monkeypatch, "scout_hub", 'label = "Scout Hub"\n') + assert load_role("scout_hub").calls_per_load_per_window is None + + +def test_role_rate_override_rejects_non_positive(tmp_path, monkeypatch, caplog): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\ncalls_per_load_per_window = 0\n', + ) + with caplog.at_level(logging.WARNING): + spec = load_role("scout_hub") + assert spec.calls_per_load_per_window is None + assert "calls_per_load_per_window" in caplog.text + + +def test_role_rate_override_rejects_non_int(tmp_path, monkeypatch, caplog): + _write_role( + tmp_path, monkeypatch, "scout_hub", + 'label = "Scout Hub"\ncalls_per_load_per_window = "lots"\n', + ) + with caplog.at_level(logging.WARNING): + spec = load_role("scout_hub") + assert spec.calls_per_load_per_window is None + + +def test_missing_manifest_yields_no_rate_override(tmp_path, monkeypatch): + monkeypatch.setattr(roles, "ROLES_DIR", tmp_path / "roles") + assert load_role("pi_lab").calls_per_load_per_window is None + + +def test_missing_manifest_yields_default_post_types(): + from src.agent.post_types import DEFAULT_POST_TYPES + + spec = load_role("definitely_not_a_role_dir") + assert spec.post_types == DEFAULT_POST_TYPES + + +def test_manifest_post_types_are_parsed(tmp_path, monkeypatch): + _write_role( + tmp_path, monkeypatch, "widget", + 'label = "Widget"\n' + '[[post_types]]\nname = "paper"\n' + '[[post_types]]\nname = "pitch"\ntargets = ["scout_hub"]\n', + ) + spec = load_role("widget") + assert [s.name for s in spec.post_types] == ["paper", "pitch"] + assert dict((s.name, s.targets) for s in spec.post_types)["pitch"] == frozenset( + {"scout_hub"} + ) + + +def test_manifest_unknown_post_type_is_dropped(tmp_path, monkeypatch, caplog): + caplog.set_level(logging.WARNING) + _write_role( + tmp_path, monkeypatch, "widget", + 'label = "Widget"\n' + '[[post_types]]\nname = "paper"\n' + '[[post_types]]\nname = "nonsense"\n', + ) + spec = load_role("widget") + assert [s.name for s in spec.post_types] == ["paper"] + assert "nonsense" in caplog.text + + +def test_malformed_toml_still_yields_default_post_types(tmp_path, monkeypatch): + from src.agent.post_types import DEFAULT_POST_TYPES + + _write_role(tmp_path, monkeypatch, "broken", "label = = =\n") + assert load_role("broken").post_types == DEFAULT_POST_TYPES + + diff --git a/tests/unit/test_roster_sync.py b/tests/unit/test_roster_sync.py index d2dcf6c..cebe06e 100644 --- a/tests/unit/test_roster_sync.py +++ b/tests/unit/test_roster_sync.py @@ -48,10 +48,10 @@ def test_token_for_agent_row_none_when_no_source(self, monkeypatch): # Live roster sync (_sync_roster_from_db) # --------------------------------------------------------------- -def _row(agent_id, token="xoxb-real"): +def _row(agent_id, token="xoxb-real", role="pi_lab"): return types.SimpleNamespace( agent_id=agent_id, bot_name=f"{agent_id.capitalize()}Bot", - pi_name=f"PI {agent_id}", slack_bot_token=token, + pi_name=f"PI {agent_id}", slack_bot_token=token, role=role, ) @@ -140,6 +140,56 @@ async def test_skips_active_agent_without_token(self, monkeypatch): assert "newbie" not in engine.agents assert set(engine.agents) == {"su"} + async def test_surviving_agent_that_gains_a_token_gets_a_client(self, monkeypatch): + """Regression: a roster agent provisioned AFTER startup stayed Slack-less. + + Measured 2026-08-06 on the blackbird deployment: 48 bots were installed + while the engine ran, their tokens landed in AgentRegistry, and not one + of them ever connected — ``Connected as`` stayed at the 7 that had tokens + at process start. Cause: ``main.py`` puts EVERY active agent into + ``self.agents`` regardless of token, so a later-provisioned agent is in + neither ``to_add`` nor ``to_remove``, the sync early-returns, and clients + are only ever built in the ``to_add`` loop. The docstring's promise that + "a freshly provisioned token is picked up on the next tick" held only for + an agent *entering* the roster. + """ + _patch_client(monkeypatch) + engine = _make_engine([_row("su"), _row("late")], existing_agents=["su", "late"]) + # Reproduce the startup state: on the roster, but tokenless then, so + # main.py never built it a client. + del engine.slack_clients["late"] + + await engine._sync_roster_from_db() + + assert "late" in engine.slack_clients, ( + "an agent already on the roster that later gains a token must be " + "given a Slack client without a process restart" + ) + assert engine.slack_clients["late"].bot_token == "xoxb-real" + + async def test_surviving_agent_without_a_token_gets_no_client(self, monkeypatch): + """The adopt path must not invent a client for a still-tokenless agent.""" + _patch_client(monkeypatch) + monkeypatch.setattr(slack_tokens, "get_settings", + lambda: types.SimpleNamespace(get_slack_tokens=lambda: {})) + engine = _make_engine([_row("su"), _row("late", token=None)], + existing_agents=["su", "late"]) + del engine.slack_clients["late"] + + await engine._sync_roster_from_db() + + assert "late" not in engine.slack_clients + + async def test_existing_client_is_not_rebuilt(self, monkeypatch): + """Adoption must be idempotent — no reconnect churn every 30s.""" + _patch_client(monkeypatch) + engine = _make_engine([_row("su")], existing_agents=["su"]) + before = engine.slack_clients["su"] + + await engine._sync_roster_from_db() + + assert engine.slack_clients["su"] is before + async def test_throttle_skips_within_interval(self, monkeypatch): _patch_client(monkeypatch) import time diff --git a/tests/unit/test_simulation_logic.py b/tests/unit/test_simulation_logic.py index 8645805..f628085 100644 --- a/tests/unit/test_simulation_logic.py +++ b/tests/unit/test_simulation_logic.py @@ -644,3 +644,344 @@ async def test_handover_memo_is_not_treated_as_revised_proposal(self): assert self.NAME in engine._finalized_private_channels props = [p for p in lairson.state.pending_proposals if p.thread_id == "200.000002"] assert len(props) == 1 + + +# --------------------------------------------------------------- +# mint_ts — monotonic, unique, ts-shaped ids (DB-primary store) +# --------------------------------------------------------------- + +class TestMintTs: + def test_monotonic_and_unique_under_tight_loop(self): + engine = SimulationEngine(agents=[], slack_clients={}) + ids = [engine.mint_ts() for _ in range(1000)] + floats = [float(x) for x in ids] + # Strictly increasing (so posted_at=float(ts) ordering is preserved). + # This is the regression guard for the float-precision bug: at the + # current epoch magnitude the old f"{time.time():.6f}" scheme produced + # ids that were equal (or non-increasing) once round-tripped to float. + assert all(b > a for a, b in zip(floats, floats[1:], strict=False)) + # All unique + assert len(set(ids)) == len(ids) + + def test_ids_are_ts_shaped_with_six_decimal_microseconds(self): + engine = SimulationEngine(agents=[], slack_clients={}) + ts = engine.mint_ts() + secs, _, micros = ts.partition(".") + assert secs.isdigit() + assert len(micros) == 6 and micros.isdigit() + + def test_seeded_high_water_mark_sorts_after_history(self): + import time + + engine = SimulationEngine(agents=[], slack_clients={}) + # Simulate a rebuild that saw history slightly ahead of the wall clock. + future = time.time() + 3600 + engine._ts_minter.seed_floor(future) + assert float(engine.mint_ts()) > future + + +# --------------------------------------------------------------- +# _flush_persisted — a failed flush must NOT drop conversation content +# (H1). The DB is the primary store, so a dropped batch is unrecoverable. +# --------------------------------------------------------------- + +class TestFlushPersistedFailure: + def _entry(self, ts, content): + from src.agent.message_log import LogEntry + + return LogEntry( + ts=ts, + channel="general", + sender_agent_id="su", + sender_name="subot", + content=content, + posted_at=float(ts), + ) + + @pytest.mark.asyncio + async def test_failed_flush_requeues_batch(self): + import uuid + + def failing_factory(): + raise RuntimeError("transient DB error") + + engine = SimulationEngine( + agents=[], + slack_clients={}, + session_factory=failing_factory, + simulation_run_id=uuid.uuid4(), + ) + engine._pending_persist = [ + self._entry("100.000001", "first"), + self._entry("100.000002", "second"), + ] + + await engine._flush_persisted() + + # The batch must survive for the next attempt, not vanish. + assert len(engine._pending_persist) == 2 + assert [e.ts for e in engine._pending_persist] == ["100.000001", "100.000002"] + + @pytest.mark.asyncio + async def test_requeued_batch_preserves_order_ahead_of_new_entries(self): + import uuid + + def failing_factory(): + raise RuntimeError("transient DB error") + + engine = SimulationEngine( + agents=[], + slack_clients={}, + session_factory=failing_factory, + simulation_run_id=uuid.uuid4(), + ) + engine._pending_persist = [ + self._entry("100.000001", "old-1"), + self._entry("100.000002", "old-2"), + ] + await engine._flush_persisted() + # A newer entry arrives after the failed flush re-queued the old batch; + # the re-queued batch must remain chronologically ahead of it. + engine._pending_persist.append(self._entry("100.000003", "new")) + + assert [e.ts for e in engine._pending_persist] == [ + "100.000001", + "100.000002", + "100.000003", + ] + + @pytest.mark.asyncio + async def test_no_db_clears_buffer(self): + # Without a session_factory the buffer is intentionally dropped so it + # can't grow unbounded — the re-queue path must not change that. + engine = SimulationEngine(agents=[], slack_clients={}) + engine._pending_persist = [self._entry("100.000001", "x")] + await engine._flush_persisted() + assert engine._pending_persist == [] + + +# --------------------------------------------------------------- +# Graceful shutdown (R2). A hard kill loses the in-flight turn's messages +# because the DB — not Slack — is now the durable store, so the SIGTERM path +# must (a) cut short the idle backoff and (b) leave the flush awaitable on the +# main coroutine rather than in a cancellable fire-and-forget task. +# --------------------------------------------------------------- + +class TestGracefulShutdown: + @pytest.mark.asyncio + async def test_request_stop_ends_the_loop_and_does_no_io(self): + engine = SimulationEngine(agents=[], slack_clients={}) + engine._running = True + engine.request_stop() + assert engine._running is False + assert engine._stop_event.is_set() + + @pytest.mark.asyncio + async def test_sleep_returns_early_once_stop_is_requested(self): + import asyncio + import time + + engine = SimulationEngine(agents=[], slack_clients={}) + + async def stop_soon(): + await asyncio.sleep(0.01) + engine.request_stop() + + started = time.monotonic() + # 30s is the longest idle backoff the main loop uses; without the + # stop-event wakeup this would outlast the container's stop grace period. + await asyncio.gather(engine._sleep(30), stop_soon()) + assert time.monotonic() - started < 1.0 + + @pytest.mark.asyncio + async def test_sleep_is_a_no_op_after_stop(self): + import time + + engine = SimulationEngine(agents=[], slack_clients={}) + engine.request_stop() + started = time.monotonic() + await engine._sleep(30) + assert time.monotonic() - started < 0.5 + + @pytest.mark.asyncio + async def test_stop_flushes_the_pending_buffer(self): + # stop() must drain the buffer, not just flip the flag — it is the last + # chance to persist the in-flight turn. + engine = SimulationEngine(agents=[], slack_clients={}) + flushed = [] + + async def fake_flush(force_stats=False): + flushed.append(force_stats) + engine._pending_persist.clear() + + engine._flush_persisted = fake_flush + engine._pending_persist = [self._entry("100.000001", "in-flight")] + + await engine.stop() + + assert flushed == [True] # forced final stats refresh + assert engine._pending_persist == [] + assert engine._running is False + + def _entry(self, ts, content): + from src.agent.message_log import LogEntry + + return LogEntry( + ts=ts, channel="general", sender_agent_id="su", + sender_name="subot", content=content, posted_at=float(ts), + ) + + +# --------------------------------------------------------------- +# Slack thread-parent translation. Slack threads on the *root's Slack ts*; the +# canonical thread_ts is only the same thing when the root was born on Slack. A +# thread started with Slack off has a minted root id, which Slack has never seen +# — mirroring a reply into it detaches the message or errors. +# --------------------------------------------------------------- + +class TestSlackParentTranslation: + def _engine_with_client(self): + from src.agent.agent import Agent + from tests.fakes import FakeSlackClient + + agent = Agent("su", "SuBot", "Andrew Su") + client = FakeSlackClient(agent_id="su") + return SimulationEngine(agents=[agent], slack_clients={"su": client}), client + + def _root(self, ts, *, slack_ts=None): + from src.agent.message_log import LogEntry + + return LogEntry( + ts=ts, channel="general", sender_agent_id="su", sender_name="SuBot", + content="root", posted_at=float(ts), is_bot=True, slack_ts=slack_ts, + ) + + def test_resolves_a_slack_backed_root_to_its_slack_ts(self): + engine, _ = self._engine_with_client() + # DB-origin root later mirrored: canonical id != Slack ts. + engine.message_log.append(self._root("1700000000.000000", slack_ts="1700009999.111111")) + assert engine._slack_parent_ts("1700000000.000000") == "1700009999.111111" + + def test_returns_none_for_a_db_origin_root(self): + engine, _ = self._engine_with_client() + engine.message_log.append(self._root("1700000000.000000")) # never mirrored + assert engine._slack_parent_ts("1700000000.000000") is None + + def test_falls_back_to_the_canonical_id_when_the_root_is_unknown(self): + # Root windowed out by the B2 rebuild bound: preserve pure-Slack-on + # behaviour, where the canonical id *is* the Slack ts. + engine, _ = self._engine_with_client() + assert engine._slack_parent_ts("1700000000.000000") == "1700000000.000000" + + def test_top_level_post_has_no_parent(self): + engine, _ = self._engine_with_client() + assert engine._slack_parent_ts(None) is None + + @pytest.mark.asyncio + async def test_reply_is_mirrored_against_the_roots_slack_ts(self): + engine, client = self._engine_with_client() + engine.message_log.append(self._root("1700000000.000000", slack_ts="1700009999.111111")) + + await engine._post_message("su", "general", "a reply", thread_ts="1700000000.000000") + + assert len(client.posted) == 1 + # Slack receives the root's Slack ts, never the minted canonical id. + assert client.posted[0]["thread_ts"] == "1700009999.111111" + + @pytest.mark.asyncio + async def test_reply_into_a_slackless_thread_is_not_mirrored(self): + # The mid-life-toggle case: thread started Slack-off, Slack now on. + engine, client = self._engine_with_client() + engine.message_log.append(self._root("1700000000.000000")) + + await engine._post_message("su", "general", "a reply", thread_ts="1700000000.000000") + + assert client.posted == [] # no bogus thread_ts sent to Slack + # ...but the message is still recorded in the DB-primary log. + replies = [e for e in engine.message_log._entries if e.thread_ts == "1700000000.000000"] + assert len(replies) == 1 + assert replies[0].content == "a reply" + assert replies[0].slack_ts is None + assert replies[0].slack_thread_ts is None + + @pytest.mark.asyncio + async def test_mirrored_reply_records_the_slack_parent_mapping(self): + engine, _ = self._engine_with_client() + engine.message_log.append(self._root("1700000000.000000", slack_ts="1700009999.111111")) + + await engine._post_message("su", "general", "a reply", thread_ts="1700000000.000000") + + reply = [e for e in engine.message_log._entries if e.thread_ts == "1700000000.000000"][0] + assert reply.slack_thread_ts == "1700009999.111111" + assert reply.thread_ts == "1700000000.000000" # canonical id unchanged + + +# --------------------------------------------------------------- +# _build_lab_directories — cohort gate must scope the "Other Labs' +# Recent Publications" section (runbook finding A3), not just the +# message log. +# --------------------------------------------------------------- + +class TestBuildLabDirectoriesCohortGate: + def _agent_with_pubs(self, agent_id, bot_name, pi_name, pub_line): + from src.agent.agent import Agent + + agent = Agent(agent_id, bot_name, pi_name) + agent._public_profile = ( + f"# {pi_name} Lab\n\n" + "## Recent Publications\n" + f"{pub_line}\n" + ) + return agent + + def test_lab_directory_respects_the_cohort_gate(self): + a = self._agent_with_pubs("a", "ABot", "A PI", "- A's distinctive paper on topic A") + b = self._agent_with_pubs("b", "BBot", "B PI", "- B's distinctive paper on topic B") + c = self._agent_with_pubs("c", "CBot", "C PI", "- C's distinctive paper on topic C") + + # Gate ON for A: may only see B. + a.allowed_sender_ids = {"b"} + # Gate OFF for B: sees everyone (unchanged behavior). + b.allowed_sender_ids = None + + engine = SimulationEngine(agents=[a, b, c], slack_clients={}) + engine._build_lab_directories() + + assert "B's distinctive paper on topic B" in a._lab_directory + assert "C's distinctive paper on topic C" not in a._lab_directory + + assert "A's distinctive paper on topic A" in b._lab_directory + assert "C's distinctive paper on topic C" in b._lab_directory + + +# --------------------------------------------------------------- +# _post_message — a text that strips to nothing must be suppressed, +# and the caller must be told. +# --------------------------------------------------------------- + +class TestPostMessageSuppressesEmptyText: + def _engine(self): + from src.agent.agent import Agent + su = Agent("su", "SuBot", "Andrew Su") + # slack_clients={} puts _post_message in MOCK mode: no network, but it + # still mints a ts and appends a LogEntry, which is what we are testing. + return SimulationEngine(agents=[su], slack_clients={}), su + + @pytest.mark.asyncio + async def test_text_that_strips_to_nothing_is_suppressed(self): + engine, _su = self._engine() + + posted = await engine._post_message("su", "general", "<slack_message></slack_message>") + + assert posted is False + assert engine.message_log._entries == [] + + @pytest.mark.asyncio + async def test_a_real_message_is_recorded_and_reports_true(self): + engine, _su = self._engine() + + posted = await engine._post_message("su", "general", "a real message") + + assert posted is True + assert len(engine.message_log._entries) == 1 + assert engine.message_log._entries[0].content == "a real message" diff --git a/tests/unit/test_slack_boundary.py b/tests/unit/test_slack_boundary.py new file mode 100644 index 0000000..50b8c3d --- /dev/null +++ b/tests/unit/test_slack_boundary.py @@ -0,0 +1,59 @@ +"""`slack_sdk` may be imported in exactly two modules. + +Fix 4 centralised pagination, retry and splitting inside AgentSlackClient, and +8515f65 then found the same four defects still live in files that had built their +own WebClient. A chokepoint you can walk around is not a chokepoint, so this test +is the wall. Adding a third importer is a design decision; make it deliberately +by editing ALLOWED, not accidentally by writing `from slack_sdk import WebClient` +in a route. +""" +import pathlib +import re + +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" + +ALLOWED = { + "agent/slack_client.py", # the engine's chokepoint + "services/slack_web.py", # the web/service boundary +} + +# Known limitation, stated so nobody mistakes this for airtight: the check is +# static and line-based, so `importlib.import_module("slack_sdk")` or +# `__import__` would slip past it. Neither appears anywhere in src/ today +# (verified), and a dynamic import of a transport is odd enough to notice in +# review. What this test does buy is that the ordinary way to bypass the +# boundary — writing `from slack_sdk import WebClient` in a route — is a build +# failure rather than a defect found in production. + +_IMPORT = re.compile(r"^\s*(?:from\s+slack_sdk[.\w]*\s+import|import\s+slack_sdk)", re.M) + + +def _importers() -> dict[str, list[int]]: + found: dict[str, list[int]] = {} + for path in sorted(SRC.rglob("*.py")): + rel = path.relative_to(SRC).as_posix() + lines = [ + i for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1) + if _IMPORT.match(line) + ] + if lines: + found[rel] = lines + return found + + +def test_slack_sdk_is_imported_only_at_the_two_boundaries(): + extra = {k: v for k, v in _importers().items() if k not in ALLOWED} + assert not extra, ( + "these modules import slack_sdk directly, bypassing the boundary — route " + f"them through src.services.slack_web instead: {extra}" + ) + + +def test_both_allowed_boundaries_still_exist(): + """Guard against the invariant passing because a boundary was deleted.""" + importers = _importers() + for allowed in ALLOWED: + assert allowed in importers, ( + f"{allowed} no longer imports slack_sdk — if it was removed, remove it " + "from ALLOWED too so this test keeps meaning something" + ) diff --git a/tests/unit/test_slack_client_contract.py b/tests/unit/test_slack_client_contract.py new file mode 100644 index 0000000..0a87be8 --- /dev/null +++ b/tests/unit/test_slack_client_contract.py @@ -0,0 +1,1007 @@ +"""What AgentSlackClient actually sends to Slack, and how it handles what comes back. + +`test_transport.py` checks protocol conformance and `test_thread_not_found.py` covers one +error path. Neither asserts on the *outbound call* — and that is the gap Rule S2 names: +the whole engine suite runs with NullTransport, so a client that quietly stopped sending +`thread_ts`, or stopped retrying a 429, would look identical from inside our own database. + +Every test here asserts on `RecordingSlackClient.calls`, which is evidence the call +happened, not merely that no exception escaped. + +The second half of the file covers the chokepoint itself — pagination, the >4000-char +split and inbound `thread_ts` normalisation. Those three were implemented with live-tier +coverage only, which means the 1091-test offline suite could not observe them at all: a +paginator reverted to a single page, or a split reverted to posting blind, passed the +whole offline suite. They are pinned here because that is where a mutation to them has to +die, not on a tier that needs a real workspace and four minutes. +""" + +import ast +import time +from pathlib import Path + +import pytest + +from src.agent import slack_client as slack_client_module +from src.agent.slack_client import ( + MAX_PAGES, + MAX_RETRIES, + SLACK_MAX_TEXT_CHARS, + SLACK_PAGE_LIMIT, + AgentSlackClient, + SlackListingIncomplete, + ThreadNotFound, + markdown_to_mrkdwn, + normalize_inbound_message, + split_for_slack, +) +from tests.fakes import RecordingSlackClient, _SlackResponse, slack_error + + +def _client(fake, *, visibility_lookup=None) -> AgentSlackClient: + c = AgentSlackClient(agent_id="su", bot_token="xoxb-test", + visibility_lookup=visibility_lookup) + c._client = fake # the seam connect() would fill + c._bot_user_id = "U_SU" + c._channel_name_to_id = {"general": "C_GENERAL"} + return c + + +class SequencedWebClient: + """A WebClient stand-in that answers a method with a *sequence* of responses. + + ``RecordingSlackClient`` returns the same dict for every call to a method, which + cannot express pagination at all: a client that followed + ``response_metadata.next_cursor`` and one that ignored it would both see the same + single page and look identical. Here each call pops the next scripted item, so + "page 1, then page 2, then stop" is expressible — and "page 2 fails" is too, by + scripting an exception in the sequence. + + Methods with no scripted sequence fall back to ``responses`` exactly as + ``RecordingSlackClient`` does, so the two are interchangeable for everything else. + """ + + def __init__(self, sequences=None, responses=None, errors=None): + self.calls: list[tuple[str, dict]] = [] + self._seq = {k: list(v) for k, v in (sequences or {}).items()} + self._responses = dict(responses or {}) + self._errors = {k: list(v) for k, v in (errors or {}).items()} + + def __getattr__(self, name): + if name.startswith("_"): + raise AttributeError(name) + + def _call(**kwargs): + self.calls.append((name, dict(kwargs))) + queue = self._errors.get(name) + if queue: + raise queue.pop(0) + seq = self._seq.get(name) + if seq is not None: + assert seq, f"{name} was called more times than the test scripted" + item = seq.pop(0) + if isinstance(item, BaseException): + raise item + return _SlackResponse(item) + return _SlackResponse(self._responses.get(name, {"ok": True})) + + return _call + + def calls_to(self, method: str) -> list[dict]: + return [kw for m, kw in self.calls if m == method] + + def unconsumed(self, method: str) -> int: + """Scripted responses the client never asked for — i.e. pages it skipped.""" + return len(self._seq.get(method, [])) + + +def _page(key: str, items: list, next_cursor: str = "") -> dict: + body = {"ok": True, key: items} + if next_cursor: + body["response_metadata"] = {"next_cursor": next_cursor} + return body + + +def _msg(ts: str, **extra) -> dict: + return {"ts": ts, "text": f"m{ts}", "user": "U_X", **extra} + + +@pytest.fixture(autouse=True) +def _no_real_sleep(monkeypatch): + """The retry path sleeps for Retry-After seconds. Without this the rate-limit + tests would really wait.""" + monkeypatch.setattr(time, "sleep", lambda _s: None) + + +# --- the retry path --------------------------------------------------------------- + + +def test_a_rate_limited_call_is_retried_and_then_succeeds(): + fake = RecordingSlackClient( + responses={"chat_postMessage": {"ok": True, "ts": "1.1", "channel": "C_GENERAL"}}, + errors={"chat_postMessage": [slack_error("ratelimited", retry_after=1)]}, + ) + out = _client(fake).post_message("general", "hello") + assert out and out["ts"] == "1.1" + assert len(fake.calls_to("chat_postMessage")) == 2, "the 429 was not retried" + + +def test_an_unthrottled_call_is_made_exactly_once(): + """Control for the test above. A client that always sent the request twice would + satisfy the retry assertion on its own.""" + fake = RecordingSlackClient( + responses={"chat_postMessage": {"ok": True, "ts": "1.2", "channel": "C_GENERAL"}}) + _client(fake).post_message("general", "hello") + assert len(fake.calls_to("chat_postMessage")) == 1 + + +def test_a_non_rate_limit_error_is_not_retried(): + """Retrying a `channel_not_found` just burns quota — the answer will not change.""" + fake = RecordingSlackClient( + errors={"chat_postMessage": [slack_error("channel_not_found")] * 5}) + assert _client(fake).post_message("general", "hello") is None + assert len(fake.calls_to("chat_postMessage")) == 1 + + +def test_retries_are_bounded_and_raise_a_SlackApiError(): + """A permanently rate-limited endpoint must give up, not spin forever — and it must + give up with the exception type its callers catch. + + Regression: `_call_with_retry` referred to `exc` after the loop, but Python unbinds + an `except ... as exc` name at the end of the except block. Exhausting the retries + raised UnboundLocalError instead of SlackApiError, which `post_message`'s + `except SlackApiError` does not catch — so a sustained 429 crashed the turn rather + than degrading to "not posted". That is the failure mode you get exactly when Slack + is throttling you. + """ + fake = RecordingSlackClient( + errors={"chat_postMessage": [slack_error("ratelimited", retry_after=1)] * 50}) + # post_message must degrade to None, not propagate anything. + assert _client(fake).post_message("general", "hello") is None + assert len(fake.calls_to("chat_postMessage")) == MAX_RETRIES + + # And the raw helper must raise the type callers handle. + from slack_sdk.errors import SlackApiError + fake2 = RecordingSlackClient( + errors={"conversations_history": [slack_error("ratelimited", retry_after=1)] * 50}) + c = _client(fake2) + with pytest.raises(SlackApiError): + c._call_with_retry(fake2.conversations_history, channel="C_GENERAL") + + +def test_retry_after_header_is_honoured(monkeypatch): + """The sleep must use Slack's Retry-After, not a hardcoded constant — ignoring it + is how a client gets itself rate-limited for longer.""" + slept = [] + monkeypatch.setattr(time, "sleep", lambda s: slept.append(s)) + fake = RecordingSlackClient( + responses={"chat_postMessage": {"ok": True, "ts": "1.3"}}, + errors={"chat_postMessage": [slack_error("ratelimited", retry_after=17)]}, + ) + _client(fake).post_message("general", "hi") + assert slept == [17], f"slept {slept}, expected Slack's Retry-After of 17" + + +# --- what actually goes on the wire ------------------------------------------------ + + +def test_thread_ts_is_omitted_for_a_root_and_sent_for_a_reply(): + """Both halves. Sending `thread_ts=None` explicitly would make every root post a + malformed reply; omitting it on a real reply silently un-threads the conversation. + """ + fake = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + _client(fake).post_message("general", "root") + kw = fake.calls_to("chat_postMessage")[0] + assert "thread_ts" not in kw, f"a root post carried thread_ts: {kw}" + + fake2 = RecordingSlackClient(responses={"chat_postMessage": { + "ok": True, "ts": "1.2", "message": {"thread_ts": "1700000000.000100"}}}) + _client(fake2).post_message("general", "reply", thread_ts="1700000000.000100") + assert fake2.calls_to("chat_postMessage")[0]["thread_ts"] == "1700000000.000100" + + +def test_a_channel_name_is_resolved_to_an_id_before_posting(): + """Slack accepts names for some endpoints and ids for others; the client normalises + to an id. A name reaching chat.postMessage works today and breaks on the endpoints + that do not accept one, so the normalisation is what keeps them consistent.""" + fake = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + _client(fake).post_message("general", "hi") + assert fake.calls_to("chat_postMessage")[0]["channel"] == "C_GENERAL" + # Control: an id passes straight through rather than being mangled. + fake2 = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + _client(fake2).post_message("C_OTHER", "hi") + assert fake2.calls_to("chat_postMessage")[0]["channel"] == "C_OTHER" + + +def test_markdown_is_translated_to_slack_mrkdwn_before_sending(): + """The text Slack receives is not the text we composed. Any live assertion that + compares a posted message to its source string has to know that.""" + fake = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + _client(fake).post_message("general", "a **bold** claim") + sent = fake.calls_to("chat_postMessage")[0]["text"] + assert "**bold**" not in sent, f"markdown was sent raw: {sent!r}" + assert "*bold*" in sent, sent + + +# --- autojoin, and the private-channel exception to it ------------------------------ + + +def test_autojoin_runs_for_a_public_channel(): + fake = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + _client(fake).post_message("general", "hi") + assert fake.calls_to("conversations_join") == [{"channel": "C_GENERAL"}] + + +def test_autojoin_is_skipped_for_a_known_private_channel(): + """A bot cannot self-join a private channel; trying hides an invite-path bug behind + a swallowed error. + + Control: the same client with the same lookup returning 'public' DOES join, so this + is about the visibility branch and not about autojoin being dead. + """ + fake = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + c = _client(fake, visibility_lookup=lambda cid: "collab_private") + c.post_message("C_PRIV", "hi") + assert fake.calls_to("conversations_join") == [] + + fake2 = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + c2 = _client(fake2, visibility_lookup=lambda cid: "public") + c2.post_message("C_PUB", "hi") + assert fake2.calls_to("conversations_join") == [{"channel": "C_PUB"}] + + +def test_a_raising_visibility_lookup_fails_open_to_public(): + """Documented behaviour: a bad lookup must not break Slack calls.""" + fake = RecordingSlackClient(responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}) + + def _boom(_cid): + raise RuntimeError("lookup exploded") + + c = _client(fake, visibility_lookup=_boom) + assert c.post_message("C_X", "hi") is not None + assert fake.calls_to("conversations_join") == [{"channel": "C_X"}] + + +def test_a_failing_autojoin_does_not_stop_the_post(): + """Autojoin is best-effort: an already-a-member bot gets an error here every time.""" + fake = RecordingSlackClient( + responses={"chat_postMessage": {"ok": True, "ts": "1.1"}}, + errors={"conversations_join": [slack_error("already_in_channel")]}, + ) + assert _client(fake).post_message("general", "hi") is not None + assert len(fake.calls_to("chat_postMessage")) == 1 + + +# --- the silent orphan: Slack drops thread_ts when the parent is gone ---------------- + + +def test_a_silently_dropped_thread_is_deleted_and_reported(): + """Slack accepts chat.postMessage against a deleted parent, drops the thread_ts, + and creates a TOP-LEVEL message. Left alone every dead root spawns a cascade of + pseudo-roots that other agents then treat as fresh posts. + + Three things must happen, and only asserting the exception would miss the worst of + them — the orphan staying in the channel. + """ + fake = RecordingSlackClient(responses={"chat_postMessage": { + "ok": True, "ts": "9.9", "channel": "C_GENERAL", + "message": {}, # no thread_ts echoed back + }}) + with pytest.raises(ThreadNotFound): + _client(fake).post_message("general", "reply", thread_ts="1.0") + assert fake.calls_to("chat_delete") == [{"channel": "C_GENERAL", "ts": "9.9"}], ( + "the orphaned top-level post was left in the channel" + ) + + +def test_a_correctly_threaded_reply_is_not_deleted(): + """Control for the test above: a client that deleted every reply would pass it.""" + fake = RecordingSlackClient(responses={"chat_postMessage": { + "ok": True, "ts": "9.9", "channel": "C_GENERAL", + "message": {"thread_ts": "1.0"}, + }}) + out = _client(fake).post_message("general", "reply", thread_ts="1.0") + assert out and out["ts"] == "9.9" + assert fake.calls_to("chat_delete") == [] + + +def test_thread_not_found_from_slack_is_raised_not_swallowed(): + fake = RecordingSlackClient( + errors={"chat_postMessage": [slack_error("thread_not_found")]}) + with pytest.raises(ThreadNotFound): + _client(fake).post_message("general", "reply", thread_ts="1.0") + + +def test_thread_not_found_on_a_ROOT_post_is_not_raised(): + """Control: the ThreadNotFound branch is conditional on thread_ts. Without this a + client that raised on every error would pass the test above.""" + fake = RecordingSlackClient( + errors={"chat_postMessage": [slack_error("thread_not_found")]}) + assert _client(fake).post_message("general", "root") is None + + +# --- not connected ------------------------------------------------------------------ + + +def test_an_unconnected_client_returns_none_rather_than_a_fake_ts(): + """The engine mints a unique canonical id when post_message returns None. A + hardcoded ts here would collide across agents and, under idempotent append, silently + drop real messages. + """ + c = AgentSlackClient(agent_id="su", bot_token="xoxb-test") + assert c._client is None + assert c.post_message("general", "hi") is None + assert c.is_connected is False + + +def test_connect_refuses_a_placeholder_token(): + c = AgentSlackClient(agent_id="su", bot_token="xoxb-placeholder-su") + assert c.connect() is False + assert c.is_connected is False + # Control: an empty token is also refused, and neither leaves a half-built client. + assert AgentSlackClient(agent_id="su", bot_token="").connect() is False + + +# =========================================================================== +# The chokepoint, at the source level +# =========================================================================== + + +def _chokepoint_nodes(): + """(direct attribute accesses on self._client, dynamic getattr lookups).""" + tree = ast.parse(Path(slack_client_module.__file__).read_text()) + + def _is_self_client(node) -> bool: + return ( + isinstance(node, ast.Attribute) + and node.attr == "_client" + and isinstance(node.value, ast.Name) + and node.value.id == "self" + ) + + owner: dict[int, str] = {} + for fn in ast.walk(tree): + if isinstance(fn, ast.FunctionDef): + for sub in ast.walk(fn): + owner.setdefault(id(sub), fn.name) + + direct = [ + (n.lineno, n.attr) for n in ast.walk(tree) + if isinstance(n, ast.Attribute) and _is_self_client(n.value) + ] + dynamic = [ + (n.lineno, owner.get(id(n), "<module>")) for n in ast.walk(tree) + if isinstance(n, ast.Call) + and isinstance(n.func, ast.Name) and n.func.id == "getattr" + and n.args and _is_self_client(n.args[0]) + ] + return direct, dynamic + + +def test_no_slack_endpoint_is_reached_outside_the_chokepoint(): + """The module docstring's central claim, asserted instead of merely written down. + + Four separate defects — `list_channels` not paginating, `create_channel` skipping + the retry, the >4000-char split, and `thread_ts == ts` normalised in one ingest path + but not another — were four instances of one structural absence: each call site + reached `self._client.<endpoint>(...)` itself and therefore had to remember the + cross-cutting rules on its own. `_api` takes the endpoint's *name* so there is + exactly one place that touches the WebClient, and a fifth instance of that class of + bug requires editing this test first. + + Parsed from the file the loaded module was imported from, not from a path guess, so + it also fails if the module under test is not the one in the working tree. + """ + assert Path(slack_client_module.__file__).is_file(), slack_client_module.__file__ + direct, dynamic = _chokepoint_nodes() + assert direct == [], ( + "these lines call a Slack endpoint directly and so inherit neither the " + f"rate-limit retry nor pagination: {direct}" + ) + assert len(dynamic) == 1, ( + f"expected exactly one dynamic endpoint lookup (in _api); found {dynamic}" + ) + assert dynamic[0][1] == "_api", ( + f"the WebClient is reached from {dynamic[0][1]}, not from _api" + ) + + +def test_every_cursor_paginated_read_goes_through_paginate(): + """Control for the test above: routing through `_api` is necessary but not + sufficient. A `_api("conversations_list", ...)` call that skipped `_paginate` would + satisfy the chokepoint test and still return one page — which is defect 1 exactly. + """ + tree = ast.parse(Path(slack_client_module.__file__).read_text()) + paginated = {"conversations_list", "conversations_history", "conversations_replies"} + offenders = [] + for fn in ast.walk(tree): + if not isinstance(fn, ast.FunctionDef) or fn.name in ("_paginate", "_api"): + continue + for call in ast.walk(fn): + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "_api" + and call.args + and isinstance(call.args[0], ast.Constant) + and call.args[0].value in paginated + ): + offenders.append((call.lineno, fn.name, call.args[0].value)) + assert offenders == [], ( + "a cursor-paginated endpoint is called outside _paginate, so it returns one " + f"page and silently drops the rest: {offenders}" + ) + + +# =========================================================================== +# Pagination — defect 1 +# =========================================================================== + + +def test_list_channels_follows_next_cursor_to_the_end(): + """The production defect: one 200-item page of a 323-channel workspace, and Slack + orders conversations.list by channel id, which is not monotonic in creation time — + so which channels a single page showed was effectively random. + """ + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "a", "id": "C_A"}], next_cursor="cur1"), + _page("channels", [{"name": "b", "id": "C_B"}], next_cursor="cur2"), + _page("channels", [{"name": "c", "id": "C_C"}]), + ]}) + c = _client(fake) + assert c.list_channels() == {"a": "C_A", "b": "C_B", "c": "C_C"} + assert fake.unconsumed("conversations_list") == 0, "pages were left unread" + assert [kw.get("cursor") for kw in fake.calls_to("conversations_list")] == [ + None, "cur1", "cur2", + ], "the cursor was not threaded through the walk" + # And the whole listing is cached, which is what makes name resolution work. + assert c._channel_name_to_id["c"] == "C_C" + + +def test_a_single_page_listing_makes_exactly_one_call(): + """Control for the test above: a client that always asked for a second page would + satisfy it, and would double every listing's cost.""" + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "a", "id": "C_A"}]), + ]}) + assert _client(fake).list_channels() == {"a": "C_A"} + assert len(fake.calls_to("conversations_list")) == 1 + + +def test_an_empty_page_carrying_a_cursor_is_followed_not_read_as_the_end(): + """Slack does return empty pages mid-walk. Stopping on one loses the tail.""" + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [], next_cursor="cur1"), + _page("channels", [{"name": "b", "id": "C_B"}]), + ]}) + assert _client(fake).list_channels() == {"b": "C_B"} + + +def test_every_page_asks_for_slacks_maximum_page_size(): + """A smaller page multiplies the round trips, and the rate limit is per method.""" + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "a", "id": "C_A"}], next_cursor="cur1"), + _page("channels", [{"name": "b", "id": "C_B"}]), + ]}) + _client(fake).list_channels() + assert [kw["limit"] for kw in fake.calls_to("conversations_list")] == [ + SLACK_PAGE_LIMIT, SLACK_PAGE_LIMIT, + ] + + +def test_a_listing_that_failed_part_way_raises_instead_of_returning_a_subset(): + """The distinction the whole exception exists for: "I got 1 of an unknown number of + channels" must never be indistinguishable from "there is 1 channel". The subset is + what made `_ensure_seeded_channels` re-create a channel Slack already had. + """ + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "a", "id": "C_A"}], next_cursor="cur1"), + slack_error("internal_error"), + ]}) + c = _client(fake) + with pytest.raises(SlackListingIncomplete) as exc: + c.list_channels() + assert [ch["name"] for ch in exc.value.partial] == ["a"] + # Caching what it *did* see is still additive and correct — only the return lies. + assert c._channel_name_to_id["a"] == "C_A" + + +def test_a_first_page_failure_raises_the_error_callers_already_handle(): + """"The request did not work at all" must keep its existing shape, or every caller's + `except SlackApiError` stops firing.""" + from slack_sdk.errors import SlackApiError + + fake = SequencedWebClient(sequences={"conversations_list": [slack_error("invalid_auth")]}) + c = _client(fake) + with pytest.raises(SlackApiError): + c._paginate("conversations_list", "channels") + # list_channels keeps degrading to {} for that case, as it always did. + fake2 = SequencedWebClient(sequences={"conversations_list": [slack_error("invalid_auth")]}) + assert _client(fake2).list_channels() == {} + + +def test_a_repeated_cursor_stops_the_walk(): + """Observed live: Slack hands back a cursor it has already issued. Following it is + an infinite loop that never returns to the caller.""" + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "a", "id": "C_A"}], next_cursor="same"), + _page("channels", [{"name": "b", "id": "C_B"}], next_cursor="same"), + ]}) + with pytest.raises(SlackListingIncomplete) as exc: + _client(fake).list_channels() + assert "repeated cursor" in exc.value.reason + assert len(fake.calls_to("conversations_list")) == 2 + + +def test_the_page_walk_is_bounded_even_if_slack_never_repeats_a_cursor(): + """The repeat check catches one cycle; only a bound guarantees termination when a + backend cycles through several distinct cursors.""" + + class _Endless: + def __init__(self): + self.n = 0 + + def conversations_list(self, **kwargs): + self.n += 1 + return _SlackResponse({ + "ok": True, + "channels": [{"name": f"c{self.n}", "id": f"C{self.n}"}], + "response_metadata": {"next_cursor": f"cur{self.n}"}, + }) + + fake = _Endless() + with pytest.raises(SlackListingIncomplete) as exc: + _client(fake).list_channels() + assert fake.n == MAX_PAGES + assert len(exc.value.partial) == MAX_PAGES + + +def test_exclude_archived_defaults_to_false_because_an_archived_channel_owns_its_name(): + """Both callers ask this question to learn whether a *name* is in use, and Slack + keeps an archived channel's name reserved. Hiding archived channels would send + `_ensure_seeded_channels` to conversations.create for a name Slack answers + `name_taken` — the same production failure, reached by a different route. + """ + fake = SequencedWebClient(sequences={"conversations_list": [_page("channels", [])]}) + _client(fake).list_channels() + kw = fake.calls_to("conversations_list")[0] + assert kw["exclude_archived"] is False + assert kw["types"] == "public_channel" + + # Control: the parameter is not inert, and include_private widens the types. + fake2 = SequencedWebClient(sequences={"conversations_list": [_page("channels", [])]}) + _client(fake2).list_channels(include_private=True, exclude_archived=True) + kw2 = fake2.calls_to("conversations_list")[0] + assert kw2["exclude_archived"] is True + assert kw2["types"] == "public_channel,private_channel" + + +def test_resolving_a_channel_name_survives_an_incomplete_listing(): + """`_resolve_channel_id` is on the post path, so it must degrade to "resolve from + what we know" rather than propagate — otherwise a partial listing turns every post + into an exception.""" + fake = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "funding", "id": "C_FUND"}], next_cursor="cur1"), + slack_error("internal_error"), + ]}) + c = _client(fake) + assert c._resolve_channel_id("funding") == "C_FUND" + + # A name that is in neither the partial listing nor the cache still falls back to + # the name itself, as it always did — no exception reaches the post path. + fake2 = SequencedWebClient(sequences={"conversations_list": [ + _page("channels", [{"name": "funding", "id": "C_FUND"}], next_cursor="cur1"), + slack_error("internal_error"), + ]}) + assert _client(fake2)._resolve_channel_id("nope") == "nope" + + +def test_polling_a_channel_pages_and_still_returns_oldest_first(): + """`limit` is Slack's *page* size, which is what Slack's `limit` means. A tick that + found more than `limit` new messages used to get the newest `limit` of them, and the + caller then advanced its poll cursor past the ones it never saw — a silent, + permanent loss. + + The page shape here is Slack's real one, measured live: with `oldest` set, + conversations.history anchors at `oldest` and pages FORWARD in time, so page 1 is the + OLDEST block (newest-first *within* the page). Reversing the concatenated walk — what + a single page needed — therefore assembles the blocks backwards, and + `_poll_slack_for_pi_messages` advances `_poll_cursors` to the last message it + iterates, so the cursor lands mid-window and the same messages are re-polled and + re-handled on every later tick. + """ + fake = SequencedWebClient(sequences={"conversations_history": [ + _page("messages", [_msg("3.0"), _msg("2.0")], next_cursor="cur1"), + _page("messages", [_msg("5.0"), _msg("4.0")]), + ]}) + out = _client(fake).poll_channel_messages("C_GENERAL", oldest="1.0", limit=2) + assert [m["ts"] for m in out] == ["2.0", "3.0", "4.0", "5.0"], ( + "a page was dropped, or the pages were assembled in the wrong order — the " + "caller's poll cursor is set from the LAST element, so it must be the newest" + ) + calls = fake.calls_to("conversations_history") + assert [kw["limit"] for kw in calls] == [2, 2] + assert {kw["oldest"] for kw in calls} == {"1.0"}, "the window moved between pages" + + +def test_polling_is_oldest_first_for_backward_paging_too(): + """Control, and the other half of the measurement: with no `oldest`, the same + endpoint pages BACKWARDS in time. Ordering by ts is correct for both, which is why it + does not depend on Slack's page order at all.""" + fake = SequencedWebClient(sequences={"conversations_history": [ + _page("messages", [_msg("5.0"), _msg("4.0")], next_cursor="cur1"), + _page("messages", [_msg("3.0"), _msg("2.0")]), + ]}) + out = _client(fake).poll_channel_messages("C_GENERAL") + assert [m["ts"] for m in out] == ["2.0", "3.0", "4.0", "5.0"] + + +def test_a_message_with_an_unparseable_ts_does_not_break_the_ordering(): + """Degrade, don't crash: a malformed ts sorts first rather than raising inside the + poll loop, which would take down the tick.""" + fake = SequencedWebClient(sequences={"conversations_history": [ + _page("messages", [_msg("2.0"), {"ts": "", "text": "odd"}]), + ]}) + assert [m["ts"] for m in _client(fake).poll_channel_messages("C_GENERAL")] == ["", "2.0"] + + +def test_an_incomplete_poll_returns_nothing_rather_than_a_partial_window(): + """The caller advances `_poll_cursors` to the last ts it is handed. Handing it a + partial window makes it step over the gap permanently, so the only safe answer is + "nothing new".""" + fake = SequencedWebClient(sequences={"conversations_history": [ + _page("messages", [_msg("5.0")], next_cursor="cur1"), + slack_error("internal_error"), + ]}) + assert _client(fake).poll_channel_messages("C_GENERAL") == [] + + +def test_full_channel_history_uses_the_partial_because_the_db_is_primary(): + """The deliberate asymmetry with the poll above. History pages newest-first, so a + partial history is missing its OLDEST messages — which the DB rebuild already has — + and the cursor derived from it still ends at the newest message. + """ + fake = SequencedWebClient(sequences={"conversations_history": [ + _page("messages", [_msg("9.0"), _msg("8.0")], next_cursor="cur1"), + slack_error("internal_error"), + ]}) + out = _client(fake).get_full_channel_history("C_GENERAL") + assert [m["ts"] for m in out] == ["8.0", "9.0"] + + +# =========================================================================== +# Inbound thread_ts normalisation — defect 4 +# =========================================================================== + +# How Slack marks a thread parent once it has replies, and one real reply as a control. +_ROOT = {"ts": "1.0", "thread_ts": "1.0", "text": "root", "user": "U_X", "reply_count": 1} +_REPLY = {"ts": "2.0", "thread_ts": "1.0", "text": "reply", "user": "U_Y"} + + +def test_normalize_inbound_message_nulls_only_a_self_referential_thread_ts(): + assert normalize_inbound_message(dict(_ROOT))["thread_ts"] is None + assert normalize_inbound_message(dict(_REPLY))["thread_ts"] == "1.0" + # A root that has no replies yet carries no thread_ts at all — left alone. + assert normalize_inbound_message({"ts": "3.0"}).get("thread_ts") is None + + +@pytest.mark.parametrize("read", [ + "poll_channel_messages", "get_full_channel_history", +]) +def test_a_channel_read_never_reports_a_root_as_a_reply_to_itself(read): + """Slack sets `thread_ts == ts` on a parent once it has replies, so a history page + hands back thread roots that look like replies to themselves. Anything treating a + non-null `thread_ts` as "this is a reply" then loses the root: + `MessageLog.get_new_top_level_posts` skips it, so it never reaches Phase 2, and the + next `_rebuild_state_from_db` makes that permanent. + """ + fake = SequencedWebClient(sequences={"conversations_history": [ + _page("messages", [dict(_REPLY), dict(_ROOT)]), + ]}) + by_ts = {m["ts"]: m for m in getattr(_client(fake), read)("C_GENERAL")} + assert by_ts["1.0"]["thread_ts"] is None, ( + "the thread root was ingested as a reply to itself, so Phase 2 will never see it" + ) + assert by_ts["2.0"]["thread_ts"] == "1.0", "a real reply lost its parent" + + +@pytest.mark.parametrize("read,args", [ + ("get_thread_replies", ("C_GENERAL", "1.0")), + ("get_all_thread_replies", ("C_GENERAL", "1.0")), +]) +def test_a_thread_read_never_reports_the_parent_as_a_reply_to_itself(read, args): + """conversations.replies returns the parent first, and it carries the same + self-referential `thread_ts`. The rule has to hold for all four inbound reads or it + is back to being a property of the call site.""" + fake = SequencedWebClient(sequences={"conversations_replies": [ + _page("messages", [dict(_ROOT), dict(_REPLY)]), + ]}) + out = getattr(_client(fake), read)(*args) + assert out[0]["ts"] == "1.0" and out[0]["thread_ts"] is None + assert out[1]["thread_ts"] == "1.0" + + +def test_workspace_bookkeeping_is_dropped_from_every_inbound_read(): + """`channel_join` and friends are not conversation. Filtering them in two of the + four reads (which is what the code did) leaks them into the thread paths.""" + fake = SequencedWebClient(sequences={"conversations_replies": [ + _page("messages", [dict(_ROOT), {"ts": "2.5", "subtype": "channel_join"}, dict(_REPLY)]), + ]}) + assert [m["ts"] for m in _client(fake).get_all_thread_replies("C_GENERAL", "1.0")] == [ + "1.0", "2.0", + ] + + +# =========================================================================== +# The >4000-char split — defect 2 +# =========================================================================== + +_LONG = "kinetics " * 600 # 5400 chars: two chunks +_VERY_LONG = "kinetics " * 1000 # 9000 chars: three chunks + + +def test_text_at_the_limit_is_left_as_one_message(): + """Measured live: 4000 characters arrive as a single Slack message. Splitting at the + boundary would turn one post into two for nothing.""" + body = "x" * SLACK_MAX_TEXT_CHARS + assert split_for_slack(body) == [body] + assert split_for_slack("short") == ["short"] + + +def test_no_chunk_exceeds_the_limit_before_or_after_mrkdwn_conversion(): + """`markdown_to_mrkdwn` runs *after* the split, so the guarantee has to survive it. + It never lengthens a string — `**x**`->`*x*` shortens, `- `->`• ` is the same + character count — and this is what says so. + """ + bodies = [ + _LONG, + _VERY_LONG, + "**bold** and - bullets\n" * 500, + "x" * 12000, # one unbreakable run + ("word " * 200 + "\n\n") * 12, # paragraph boundaries + "a\n" * 5000, # line boundaries only + "Sentence one. Sentence two. " * 400, # sentence boundaries + "```\n" + "row = measure()\n" * 500 + "```", # a fenced block + ] + for body in bodies: + chunks = split_for_slack(body) + assert chunks, body[:40] + over = [len(c) for c in chunks if len(c) > SLACK_MAX_TEXT_CHARS] + assert not over, f"chunk(s) over the limit: {over} for {body[:40]!r}" + after = [len(markdown_to_mrkdwn(c)) for c in chunks] + assert max(after) <= SLACK_MAX_TEXT_CHARS, ( + f"mrkdwn conversion pushed a chunk over the limit: {max(after)}" + ) + + +def test_the_split_loses_and_duplicates_no_content(): + """Compared with whitespace removed, because cuts land on whitespace and each chunk + is stripped. A cut that ate a character, or repeated one, fails here.""" + for body in (_LONG, _VERY_LONG, "Sentence one. Sentence two. " * 400): + joined = "".join(split_for_slack(body)) + assert joined.replace(" ", "").replace("\n", "") == ( + body.replace(" ", "").replace("\n", "") + ) + + +def test_a_cut_lands_on_a_boundary_rather_than_inside_a_word(): + chunks = split_for_slack(_LONG) + assert len(chunks) == 2, len(chunks) + assert chunks[0].endswith("kinetics"), repr(chunks[0][-20:]) + assert chunks[1].startswith("kinetics"), repr(chunks[1][:20]) + + +def test_an_unbreakable_run_is_cut_anyway_rather_than_left_over_the_limit(): + """A 12000-character token has no non-corrupting split point. Refusing to split is + not an option — Slack would split it for us and hide the tail.""" + chunks = split_for_slack("x" * 12000) + assert [len(c) for c in chunks] == [4000, 4000, 4000] + + +@pytest.mark.parametrize("limit", [1, 2, 8, 9, 20]) +def test_a_tiny_limit_terminates_instead_of_hanging_the_turn(limit): + """`limit` is a parameter, and the fence-repair reserve is 8 characters — so a + fenced body with `limit=8` left a budget of zero, `_cut_at` returned 0, `rest` never + shrank and this looped forever inside the calling turn. Measured: it never returned. + Unreachable at the module's own 4000-char limit, but a hang is not a failure mode + worth leaving available. + """ + for body in ("```\n" + "abc def\n" * 20 + "```", "abc def ghi " * 20): + chunks = split_for_slack(body, limit=limit) + assert chunks, (limit, body[:20]) + assert "".join(chunks).replace(" ", "").replace("\n", "").replace("`", "") == ( + body.replace(" ", "").replace("\n", "").replace("`", "") + ), f"content lost at limit={limit}" + + +def test_a_code_fence_spanning_a_cut_is_closed_and_reopened(): + """Slack renders `text` as mrkdwn, so a chunk ending inside a ``` block renders its + tail as code and the next chunk renders its head as prose — the block boundary + moves. Every chunk has to be balanced on its own.""" + body = "```\n" + "row = measure(sample)\n" * 400 + "```" + chunks = split_for_slack(body) + assert len(chunks) > 1 + for i, chunk in enumerate(chunks): + assert chunk.count("```") % 2 == 0, f"chunk {i} leaves a fence open" + # Control: unfenced text of the same shape gets no added backticks. + assert all("```" not in c for c in split_for_slack("row = measure(sample)\n" * 400)) + + +def test_an_over_limit_post_becomes_several_messages_and_reports_every_one(): + """Slack splits a >4000-char `text` itself and returns only the LAST chunk's ts. A + client that posts blind therefore records a ts naming the *tail* of its own message + and leaves every earlier chunk in Slack with no database row — and on restart + `_rebuild_state_from_slack` ingests those as brand-new inbound messages. + """ + fake = SequencedWebClient(sequences={"chat_postMessage": [ + {"ok": True, "ts": "1.0", "channel": "C_GENERAL", "message": {}}, + {"ok": True, "ts": "2.0", "channel": "C_GENERAL", "message": {"thread_ts": "1.0"}}, + ]}) + out = _client(fake).post_message("general", _LONG) + assert out and len(out["posted_messages"]) == 2, out + assert out["ts"] == "1.0", ( + "post_message returned a ts other than the FIRST message's — this is the value " + "the engine records as the canonical id and threads replies onto" + ) + sent = [kw["text"] for kw in fake.calls_to("chat_postMessage")] + assert len(sent) == 2 and all(len(t) <= SLACK_MAX_TEXT_CHARS for t in sent), ( + [len(t) for t in sent] + ) + # A split root stays ONE top-level post: the continuation hangs off the first + # message, so nobody else's Phase 2 scan sees two roots where one post was written. + assert "thread_ts" not in fake.calls_to("chat_postMessage")[0] + assert fake.calls_to("chat_postMessage")[1]["thread_ts"] == "1.0" + posted = out["posted_messages"] + assert posted[0]["thread_ts"] is None + assert posted[1]["thread_ts"] == "1.0" + # Each record carries the SOURCE text of its own message, which is what the DB + # stores for it — that is what puts agent_messages in bijection with Slack. + assert [p["text"] for p in posted] == split_for_slack(_LONG) + + +def test_a_post_within_the_limit_is_one_message_reported_as_one(): + """Control: a client that split everything, or that reported a phantom second + message, would pass the test above.""" + fake = SequencedWebClient(sequences={"chat_postMessage": [ + {"ok": True, "ts": "1.0", "channel": "C_GENERAL", "message": {}}, + ]}) + out = _client(fake).post_message("general", "short enough") + assert out["posted_messages"] == [ + {"ts": "1.0", "channel": "C_GENERAL", "text": "short enough", "thread_ts": None}, + ] + assert len(fake.calls_to("chat_postMessage")) == 1 + + +def test_an_over_limit_reply_keeps_every_chunk_in_the_callers_thread(): + """Control for the root case: for a *reply* every chunk belongs to the thread the + caller named, not to a sub-thread on the first chunk.""" + fake = SequencedWebClient(sequences={"chat_postMessage": [ + {"ok": True, "ts": f"{i}.0", "channel": "C_GENERAL", "message": {"thread_ts": "0.5"}} + for i in range(1, 4) + ]}) + out = _client(fake).post_message("general", _VERY_LONG, thread_ts="0.5") + posted = out["posted_messages"] + assert len(posted) == 3 + assert all(p["thread_ts"] == "0.5" for p in posted), [p["thread_ts"] for p in posted] + assert {kw["thread_ts"] for kw in fake.calls_to("chat_postMessage")} == {"0.5"} + + +def test_a_chunk_that_fails_stops_the_rest_and_reports_only_what_landed(): + """Never post the tail of a message whose head failed, and never claim a message + that Slack refused. The caller records one row per reported message, so an + over-report is a phantom row and an under-report is a lost one.""" + fake = SequencedWebClient(sequences={"chat_postMessage": [ + {"ok": True, "ts": "1.0", "channel": "C_GENERAL", "message": {}}, + slack_error("msg_too_long"), + ]}) + out = _client(fake).post_message("general", _VERY_LONG) + assert len(out["posted_messages"]) == 1 and out["ts"] == "1.0" + assert len(fake.calls_to("chat_postMessage")) == 2, "it kept going after a failure" + + # And a failure on the FIRST chunk posts nothing at all. + fake2 = SequencedWebClient(sequences={"chat_postMessage": [slack_error("msg_too_long")]}) + assert _client(fake2).post_message("general", _VERY_LONG) is None + + +def test_the_recorded_thread_parent_is_the_one_slack_reports(): + """Not the one we asked for. The row has to describe the message Slack actually + made, or the mirror mapping is a guess.""" + fake = SequencedWebClient(sequences={"chat_postMessage": [ + {"ok": True, "ts": "9.9", "channel": "C_OTHER", "message": {"thread_ts": "0.5"}}, + ]}) + out = _client(fake).post_message("general", "reply", thread_ts="0.5") + assert out["posted_messages"] == [ + {"ts": "9.9", "channel": "C_OTHER", "text": "reply", "thread_ts": "0.5"}, + ] + + +def test_handover_post_budget_stays_under_the_slack_split_threshold(): + """_add_handover_message writes one DB row per call, not one per Slack message. + + That is only correct while every post it makes fits in a single Slack + message. 8515f65 recorded that raising _MAX_POST_CHARS silently reinstates + defect 2 — the DB and Slack disagreeing about how many messages exist — and + nothing pinned the coupling. This is that pin. If you need a bigger budget, + make _add_handover_message honour posted_messages first, then delete this. + """ + from src.services.private_channels import _MAX_POST_CHARS + + assert _MAX_POST_CHARS < SLACK_MAX_TEXT_CHARS, ( + f"_MAX_POST_CHARS={_MAX_POST_CHARS} would let a handover post split into " + f"multiple Slack messages (limit {SLACK_MAX_TEXT_CHARS}), while " + "_add_handover_message still writes exactly one DB row per call" + ) + + +# =========================================================================== +# create_channel through the chokepoint — defect 3 +# =========================================================================== + + +def test_a_rate_limited_channel_create_is_retried(): + """It bypassed `_call_with_retry` entirely, so a 429 collapsed into the same `None` + that means "Slack refused" — and `_ensure_seeded_channels` left the channel with no + id at all.""" + fake = SequencedWebClient(sequences={"conversations_create": [ + slack_error("ratelimited", retry_after=1), + {"ok": True, "channel": {"id": "C_NEW", "name": "seeded"}}, + ]}) + c = _client(fake) + assert c.create_channel("seeded") == {"id": "C_NEW", "name": "seeded"} + assert len(fake.calls_to("conversations_create")) == 2, "the 429 was not retried" + assert c._channel_name_to_id["seeded"] == "C_NEW" + + +def test_an_unthrottled_channel_create_is_made_exactly_once(): + """Control for the test above.""" + fake = SequencedWebClient(sequences={"conversations_create": [ + {"ok": True, "channel": {"id": "C_NEW", "name": "seeded"}}, + ]}) + _client(fake).create_channel("seeded") + assert len(fake.calls_to("conversations_create")) == 1 + + +def test_name_taken_adopts_the_existing_channel_rather_than_reporting_failure(): + """`name_taken` means the channel exists — an archived one still owns its name — so + reporting failure is what left `_channel_id_map[name] = None`, after which every + post to it was addressed by name and Slack answered `not_in_channel`.""" + fake = SequencedWebClient( + sequences={ + "conversations_create": [slack_error("name_taken")], + "conversations_list": [_page("channels", [{"name": "seeded", "id": "C_OLD"}])], + }, + ) + assert _client(fake).create_channel("seeded") == {"id": "C_OLD", "name": "seeded"} + + +def test_name_taken_on_an_invisible_channel_still_reports_failure(): + """Control: adoption is not unconditional. A private channel this bot cannot see is + `name_taken` with no id to adopt, and inventing one would be worse than failing.""" + fake = SequencedWebClient( + sequences={ + "conversations_create": [slack_error("name_taken")], + "conversations_list": [_page("channels", [])], + }, + ) + assert _client(fake).create_channel("seeded") is None + + +def test_a_private_channel_create_is_also_retried_on_a_429(): + fake = SequencedWebClient(sequences={"conversations_create": [ + slack_error("ratelimited", retry_after=1), + {"ok": True, "channel": {"id": "G_NEW", "name": "priv-a-b-x"}}, + ]}) + out = _client(fake).create_private_channel("priv-a-b") + assert out["id"] == "G_NEW" + assert len(fake.calls_to("conversations_create")) == 2 + + +def test_an_unconnected_client_raises_rather_than_calling_a_missing_endpoint(): + """`_api` with no WebClient behind it is a programming error, not a runtime + condition: every public method guards on `self._client` first. Making it explicit + keeps a new method that forgets the guard from failing as an AttributeError.""" + from src.agent.slack_client import SlackNotConnected + + c = AgentSlackClient(agent_id="su", bot_token="xoxb-test") + with pytest.raises(SlackNotConnected): + c._api("auth_test") diff --git a/tests/unit/test_slack_provisioning.py b/tests/unit/test_slack_provisioning.py new file mode 100644 index 0000000..9bdce80 --- /dev/null +++ b/tests/unit/test_slack_provisioning.py @@ -0,0 +1,249 @@ +"""Slack app provisioning: the manifest we submit, token rotation, and secret hygiene. + +`test_admin_provisioning.py` has three tests covering the happy path of the admin +service. Nothing covered the manifest contents, `rotate_config_token`, `exchange_code`, +or the caching that keeps a single-use refresh token from being burned on every click. + +The manifest test is the load-bearing one: a scope missing from `BOT_SCOPES` produces a +bot that provisions cleanly, connects cleanly, and then fails one specific API call at +runtime — and fixing it needs a manifest change *and* a manual reinstall of every bot. +""" + +import time + +import httpx +import pytest +from sqlalchemy import select + +from src.models import AppSetting +from src.services.admin_provisioning import _config_token +from src.services.slack_provisioning import BOT_SCOPES, create_app, exchange_code + +pytestmark = pytest.mark.integration + + +class _Resp: + def __init__(self, payload): + self._payload = payload + + def json(self): + return self._payload + + +# --- the manifest ------------------------------------------------------------------- + +# Every Slack API method the codebase calls, and the bot scope it needs. Derived by +# grepping src/ for `client.<method>` — see the Surface Inventory in +# .notes/slack-integration-test-plan.md. +METHOD_SCOPES = { + "auth.test": None, # no scope required + "chat.postMessage": "chat:write", + "chat.delete": "chat:write", + "conversations.list (public)": "channels:read", + "conversations.list (private)": "groups:read", + "conversations.create (public)": "channels:manage", + "conversations.create (private)": "groups:write", + "conversations.join": "channels:join", + "conversations.invite (private)": "groups:write", + "conversations.history (public)": "channels:history", + "conversations.history (private)": "groups:history", + "conversations.replies (public)": "channels:history", + "conversations.open": "im:write", + "conversations.history (dm)": "im:history", + "users.info": "users:read", + "users.lookupByEmail": "users:read.email", +} + + +def test_manifest_requests_every_scope_the_client_actually_needs(): + """The invariant that keeps provisioning honest. + + AgentSlackClient exposes create_private_channel() and invite_to_channel(), and + private-channel migration calls both. Both need `groups:write`. A scope absent here + is invisible until the one call that needs it fails at runtime with missing_scope, + on a bot that otherwise looks perfectly healthy. + """ + needed = {s for s in METHOD_SCOPES.values() if s} + missing = sorted(needed - set(BOT_SCOPES)) + assert not missing, ( + f"BOT_SCOPES is missing {missing}. Methods that need them: " + + ", ".join(m for m, s in METHOD_SCOPES.items() if s in missing) + ) + + +def test_method_scope_table_is_not_trivially_satisfiable(): + """Control for the table above: it must name scopes that are genuinely required, + not an empty set. An empty table would make the invariant vacuous.""" + assert len({s for s in METHOD_SCOPES.values() if s}) >= 8 + + +def test_create_app_manifest_shape(monkeypatch): + captured = {} + + def _post(url, **kw): + captured["url"] = url + captured["json"] = kw.get("json") + captured["auth"] = kw.get("headers", {}).get("Authorization") + return _Resp({"ok": True, "app_id": "A1", + "credentials": {"client_id": "cid", "client_secret": "csec"}, + "oauth_authorize_url": "https://slack.com/oauth/v2/authorize?x=1"}) + + monkeypatch.setattr(httpx, "post", _post) + out = create_app("xoxe.xoxp-token", "su", "SuBot", "PI Su", + "https://example.test/admin/agents/slack/callback") + + assert captured["url"].endswith("/apps.manifest.create") + assert captured["auth"] == "Bearer xoxe.xoxp-token" + m = captured["json"]["manifest"] + assert m["display_information"]["name"] == "SuBot" + assert m["features"]["bot_user"]["display_name"] == "SuBot" + assert m["oauth_config"]["redirect_urls"] == [ + "https://example.test/admin/agents/slack/callback"] + assert set(m["oauth_config"]["scopes"]["bot"]) == set(BOT_SCOPES) + # Socket Mode off is what makes the polling design correct; org deploy off keeps + # the app single-workspace. + assert m["settings"]["socket_mode_enabled"] is False + assert m["settings"]["org_deploy_enabled"] is False + assert out == { + "agent_id": "su", "bot_name": "SuBot", "pi_name": "PI Su", "app_id": "A1", + "client_id": "cid", "client_secret": "csec", + "oauth_url": "https://slack.com/oauth/v2/authorize?x=1", + } + + +def test_create_app_retries_only_on_rate_limit(monkeypatch): + """Control included: a non-rate-limit error must raise on the FIRST call, so a + create_app that retried everything would not pass both halves.""" + monkeypatch.setattr(time, "sleep", lambda _s: None) + calls = [] + + def _post_ratelimited(url, **kw): + calls.append(url) + if len(calls) < 3: + return _Resp({"ok": False, "error": "ratelimited", "retry_after": 1}) + return _Resp({"ok": True, "app_id": "A1", + "credentials": {"client_id": "c", "client_secret": "s"}, + "oauth_authorize_url": "u"}) + + monkeypatch.setattr(httpx, "post", _post_ratelimited) + assert create_app("t", "su", "SuBot", "PI", "https://x/cb")["app_id"] == "A1" + assert len(calls) == 3 + + calls.clear() + monkeypatch.setattr(httpx, "post", + lambda url, **kw: calls.append(url) or + _Resp({"ok": False, "error": "invalid_manifest"})) + with pytest.raises(RuntimeError, match="invalid_manifest"): + create_app("t", "su", "SuBot", "PI", "https://x/cb") + assert len(calls) == 1 + + +# --- secret hygiene ------------------------------------------------------------------- + + +def test_exchange_code_never_echoes_the_token(monkeypatch): + """SEC-9. This error string reaches the server log and a user-facing + ?slack_error= redirect, so any fragment of the value is a leak.""" + monkeypatch.setattr(httpx, "post", lambda *a, **k: _Resp( + {"ok": True, "access_token": "xoxp-WRONGTYPE-abcdefghijklmnop"})) + with pytest.raises(RuntimeError) as ei: + exchange_code("cid", "csec", "code", "https://example.test/cb") + msg = str(ei.value) + for fragment in ("xoxp-WRONGTYPE", "abcdefghijklmnop", "WRONGTYPE"): + assert fragment not in msg, f"the token leaked into the error: {msg!r}" + assert msg.strip(), "control leg failed: the message is empty, which is unhelpful" + + +def test_exchange_code_returns_a_bot_token(monkeypatch): + """Control for the test above: the happy path must actually work, or 'never echoes + the token' is satisfied by a function that always raises.""" + monkeypatch.setattr(httpx, "post", lambda *a, **k: _Resp( + {"ok": True, "access_token": "xoxb-good-token"})) + assert exchange_code("cid", "csec", "code", "https://x/cb") == "xoxb-good-token" + + +def test_exchange_code_surfaces_a_slack_error(monkeypatch): + monkeypatch.setattr(httpx, "post", lambda *a, **k: _Resp( + {"ok": False, "error": "invalid_code"})) + with pytest.raises(RuntimeError, match="invalid_code"): + exchange_code("cid", "csec", "code", "https://x/cb") + + +# --- config-token rotation -------------------------------------------------------- + + +async def test_rotation_persists_the_whole_triple(db_session, monkeypatch): + """SEC-10. The refresh token just spent is dead; if only some of the three KV rows + land, app-config access is lost with no way back to the old pair.""" + monkeypatch.setattr( + "src.services.slack_provisioning.rotate_config_token", + lambda refresh: ("xoxe.xoxp-NEW", "xoxe-1-NEWREFRESH", int(time.time()) + 43200), + ) + monkeypatch.setattr("src.services.admin_provisioning.get_settings", + lambda: _settings_with(refresh="xoxe-1-SEED")) + tok = await _config_token(db_session) + assert tok == "xoxe.xoxp-NEW" + rows = {r.key: r.value for r in + (await db_session.execute(select(AppSetting))).scalars().all()} + assert rows["slack_config_token"] == "xoxe.xoxp-NEW" + assert rows["slack_config_refresh_token"] == "xoxe-1-NEWREFRESH" + assert int(rows["slack_config_token_exp"]) > time.time() + + +async def test_a_cached_token_is_reused_and_does_not_rotate(db_session, monkeypatch): + """Control for the test above, and the property that makes provisioning usable at + all: rotation must be RARE. A _config_token that rotated on every call would satisfy + the atomicity test while burning a single-use refresh token on every admin click — + and a crash between Slack's rotate and our commit would strand the new pair. + """ + calls = [] + monkeypatch.setattr( + "src.services.slack_provisioning.rotate_config_token", + lambda r: (calls.append(r) or + ("xoxe.xoxp-N", "xoxe-1-N", int(time.time()) + 43200)), + ) + monkeypatch.setattr("src.services.admin_provisioning.get_settings", + lambda: _settings_with(refresh="xoxe-1-SEED")) + first = await _config_token(db_session) + second = await _config_token(db_session) + assert first == second == "xoxe.xoxp-N" + assert len(calls) == 1, f"rotated {len(calls)} times across two calls" + + +async def test_an_expiring_token_is_rotated_before_it_dies(db_session, monkeypatch): + """The cache must not hand out a token that expires mid-request.""" + db_session.add(AppSetting(key="slack_config_token", value="xoxe.xoxp-OLD")) + db_session.add(AppSetting(key="slack_config_refresh_token", value="xoxe-1-OLD")) + db_session.add(AppSetting(key="slack_config_token_exp", + value=str(int(time.time()) + 5))) # inside the margin + await db_session.flush() + monkeypatch.setattr( + "src.services.slack_provisioning.rotate_config_token", + lambda r: ("xoxe.xoxp-FRESH", "xoxe-1-FRESH", int(time.time()) + 43200)) + monkeypatch.setattr("src.services.admin_provisioning.get_settings", + lambda: _settings_with(refresh="")) + assert await _config_token(db_session) == "xoxe.xoxp-FRESH" + + +async def test_a_valid_cached_token_is_returned_untouched(db_session, monkeypatch): + """Control for the expiry test: a token with plenty of life left must NOT rotate.""" + db_session.add(AppSetting(key="slack_config_token", value="xoxe.xoxp-STILLGOOD")) + db_session.add(AppSetting(key="slack_config_token_exp", + value=str(int(time.time()) + 43200))) + await db_session.flush() + + def _boom(_r): + raise AssertionError("rotate must not be called for a healthy cached token") + + monkeypatch.setattr("src.services.slack_provisioning.rotate_config_token", _boom) + monkeypatch.setattr("src.services.admin_provisioning.get_settings", + lambda: _settings_with(refresh="xoxe-1-SEED")) + assert await _config_token(db_session) == "xoxe.xoxp-STILLGOOD" + + +def _settings_with(*, refresh: str = "", token: str = ""): + import types + return types.SimpleNamespace( + slack_config_refresh_token=refresh, slack_config_token=token, + base_url="http://localhost:8001", + ) diff --git a/tests/unit/test_slack_tokens.py b/tests/unit/test_slack_tokens.py new file mode 100644 index 0000000..e15ea7f --- /dev/null +++ b/tests/unit/test_slack_tokens.py @@ -0,0 +1,365 @@ +"""Slack token validity, source precedence, and the `slack_enabled` tri-state. + +These three decide whether the whole Slack integration is on, and which credential each +agent uses. Nothing tested them directly before: `test_roster_sync.py` exercises +`token_for_agent_row` incidentally, and the `slack_enabled` resolution in +`src/agent/main.py` had no coverage at all. + +Why the token-shape table matters more than it looks: `slack_globally_enabled()` +auto-detects Slack as ON from the mere *presence* of a "valid" token. So whatever +`is_valid_token` accepts is what can silently switch the integration on — and then fail +every API call. +""" + +import pytest + +from src.config import Settings, get_settings +from src.models import AgentRegistry +from src.services.slack_tokens import ( + env_token, + get_agent_bot_token, + get_any_bot_token, + is_valid_token, + slack_globally_enabled, + token_for_agent_row, +) +from tests import factories + +# Slack bot tokens are always `xoxb-`. Every consumer of is_valid_token passes a bot +# token, so anything else reaching it is a misconfiguration that must not turn Slack on. +TOKEN_CASES = [ + ("xoxb-1111-2222-abcdefghijklmnop", True), + ("xoxb-real-looking-token-value", True), + ("", False), + (None, False), + (" ", False), + ("\n", False), + ("xoxb-placeholder", False), + ("xoxb-placeholder-su", False), + # A USER token in the bot-token field. Accepted by the pre-hardening + # implementation, which meant one paste could flip slack_enabled on and then fail + # every call with not_allowed_token_type. + ("xoxp-EXAMPLE-NOT-A-REAL-TOKEN", False), + # A CONFIG token in the bot-token field — same hazard. This is the exact token type + # used for provisioning, so the two live side by side in the same .env. + ("xoxe.xoxp-1-EXAMPLE-NOT-A-REAL-TOKEN", False), + ("xoxe-1-EXAMPLE-NOT-A-REAL-TOKEN", False), + # Unfilled template values. + ("REPLACE_ME", False), + ("xoxb-your-token-here", True), # indistinguishable from a real token; documented +] + + +@pytest.mark.parametrize("tok,valid", TOKEN_CASES, + ids=[repr(c[0])[:26] for c in TOKEN_CASES]) +def test_is_valid_token(tok, valid): + assert is_valid_token(tok) is valid + + +def test_token_cases_have_both_polarities(): + """Control for the table: an is_valid_token that returned a constant would satisfy + an all-True or all-False table without anyone noticing.""" + assert {c[1] for c in TOKEN_CASES} == {True, False} + + +def test_the_only_accepted_shape_is_a_bot_token(): + """States the rule the table encodes, so a future edit that loosens the check has + to delete an assertion that says why rather than just flip a row.""" + for prefix in ("xoxp-", "xoxe.xoxp-", "xoxe-", "xapp-", "xoxa-", "Bearer "): + assert is_valid_token(prefix + "something") is False, prefix + assert is_valid_token("xoxb-something") is True + + +# --- source precedence: the DB column is authoritative, .env is a fallback --------- + + +def _clear_settings_cache(): + get_settings.cache_clear() + + +# Every SLACK_BOT_TOKEN_* name Settings knows about, derived rather than listed: there +# are 125 of them and the roster grows. +_ALL_BOT_TOKEN_ENV = tuple( + f.upper() for f in Settings.model_fields if f.startswith("slack_bot_token_") +) + + +def _blank_all_bot_tokens(monkeypatch): + """Make "no bot token is configured" actually true. + + Two tests here assert that nothing usable exists, and they used to defend against + exactly one ambient value — ``monkeypatch.delenv("SLACK_BOT_TOKEN_SU")`` — while + ``Settings.get_slack_tokens()`` reads 125. They passed only because .env happened to + hold none of them. Provisioning two probe bots put real tokens in .env and both went + red, on a machine where the product was working fine. + + ``delenv`` cannot fix it either: pydantic-settings reads the .env *file*, so removing + a process env var leaves the file value in place. An empty env var does override the + file (env beats .env in the precedence chain), so blanking is the lever that works. + """ + for name in _ALL_BOT_TOKEN_ENV: + monkeypatch.setenv(name, "") + + +def test_token_for_agent_row_prefers_the_db_column(monkeypatch): + """CLAUDE.md: the AgentRegistry column is the source of truth, .env is a read + fallback. Both halves, so a resolver that only ever read one source fails.""" + monkeypatch.setenv("SLACK_BOT_TOKEN_SU", "xoxb-from-the-env-file") + _clear_settings_cache() + try: + row = AgentRegistry(agent_id="su", bot_name="SuBot", pi_name="PI Su", + slack_bot_token="xoxb-from-the-database") + assert token_for_agent_row(row) == "xoxb-from-the-database" + # Control: with the column empty the env value IS used, so the assertion above + # is about precedence rather than about the fallback being dead. + row.slack_bot_token = None + assert token_for_agent_row(row) == "xoxb-from-the-env-file" + # And an invalid column value falls back rather than winning. + row.slack_bot_token = "xoxb-placeholder" + assert token_for_agent_row(row) == "xoxb-from-the-env-file" + finally: + _clear_settings_cache() + + +def test_env_token_rejects_an_invalid_env_value(monkeypatch): + monkeypatch.setenv("SLACK_BOT_TOKEN_SU", "xoxb-placeholder") + _clear_settings_cache() + try: + assert env_token("su") is None + # Control: a real-looking value comes back. + monkeypatch.setenv("SLACK_BOT_TOKEN_SU", "xoxb-good") + _clear_settings_cache() + assert env_token("su") == "xoxb-good" + finally: + _clear_settings_cache() + + +def test_env_token_for_an_unknown_agent_is_none(): + assert env_token("nobody-by-that-name") is None + + +@pytest.mark.integration +async def test_get_agent_bot_token_reads_the_db_then_env(db_session, monkeypatch): + user = await factories.make_user(db_session, email="su-tok@example.org") + agent = await factories.make_agent( + db_session, user=user, agent_id="su", bot_name="SuBot", pi_name="PI Su", + status="active", slack_bot_token="xoxb-db-value", + ) + await db_session.flush() + monkeypatch.setenv("SLACK_BOT_TOKEN_SU", "xoxb-env-value") + _clear_settings_cache() + try: + assert await get_agent_bot_token(db_session, "su") == "xoxb-db-value" + agent.slack_bot_token = None + await db_session.flush() + assert await get_agent_bot_token(db_session, "su") == "xoxb-env-value" + finally: + _clear_settings_cache() + + +@pytest.mark.integration +async def test_get_any_bot_token_ignores_invalid_rows(db_session, monkeypatch): + """A placeholder row must not satisfy 'any usable token' — that is what + auto-detect keys on, so a placeholder would switch Slack on for the deployment.""" + _blank_all_bot_tokens(monkeypatch) + _clear_settings_cache() + try: + u1 = await factories.make_user(db_session, email="a-tok@example.org") + await factories.make_agent(db_session, user=u1, agent_id="a1", bot_name="A1Bot", + status="active", slack_bot_token="xoxb-placeholder") + await db_session.flush() + assert await get_any_bot_token(db_session) is None + # Control: one real token and it is found. + u2 = await factories.make_user(db_session, email="b-tok@example.org") + await factories.make_agent(db_session, user=u2, agent_id="a2", bot_name="A2Bot", + status="active", slack_bot_token="xoxb-real") + await db_session.flush() + assert await get_any_bot_token(db_session) == "xoxb-real" + finally: + _clear_settings_cache() + + +# --- the slack_enabled tri-state (mirrors src/agent/main.py:110-114) --------------- + + +ENABLED_CASES = [ + # (name, settings value, a usable token exists, expected) + ("forced off, token present", False, True, False), + ("forced off, no token", False, False, False), + ("forced on, no token", True, False, True), + ("forced on, token present", True, True, True), + ("auto, no token", None, False, False), + ("auto, token present", None, True, True), +] + + +@pytest.mark.integration +@pytest.mark.parametrize("name,setting,has_token,expected", ENABLED_CASES, + ids=[c[0] for c in ENABLED_CASES]) +async def test_slack_globally_enabled_tri_state( + db_session, monkeypatch, name, setting, has_token, expected +): + _blank_all_bot_tokens(monkeypatch) + if setting is None: + monkeypatch.delenv("SLACK_ENABLED", raising=False) + else: + monkeypatch.setenv("SLACK_ENABLED", "true" if setting else "false") + _clear_settings_cache() + if setting is None: + # `delenv` cannot make "auto" true on this host: pydantic-settings falls + # back to the .env *file* when the process env var is absent, and the + # deployed .env sets SLACK_ENABLED=true. Absence of an env var is not + # expressible through the env layer, so pin the resolved attribute + # directly — the one lever that actually forces auto-detect. + monkeypatch.setattr(get_settings(), "slack_enabled", None) + try: + if has_token: + u = await factories.make_user(db_session, email=f"{name[:8]}@example.org") + await factories.make_agent( + db_session, user=u, agent_id="su", bot_name="SuBot", + status="active", slack_bot_token="xoxb-real", + ) + await db_session.flush() + assert await slack_globally_enabled(db_session) is expected, name + finally: + _clear_settings_cache() + + +def test_enabled_cases_cover_all_three_branches(): + """Control: the table must exercise forced-on, forced-off AND auto-detect, and + auto-detect must appear with both outcomes. Otherwise a resolver that ignored the + setting, or one that ignored the tokens, would pass.""" + assert {c[1] for c in ENABLED_CASES} == {True, False, None} + auto = {c[3] for c in ENABLED_CASES if c[1] is None} + assert auto == {True, False}, "auto-detect is only tested in one direction" + + +# --- secret redaction over the fields that exist today ---------------------------- + + +def test_every_slack_secret_is_redacted_in_the_settings_repr(): + """`test_config_secret_redaction.py` predates the config-token pair, so the two + provisioning credentials were never checked. A settings object reaches logs and + error pages; a bot or config token in there is a workspace takeover. + + Control included: a non-secret field must still be visible, so a repr that returned + nothing at all would not pass. + """ + s = Settings( + slack_bot_token_su="xoxb-secret-aaaaaaa", + slack_config_token="xoxe.xoxp-secret-bbbbbbb", + slack_config_refresh_token="xoxe-1-secret-ccccccc", + ) + text = repr(s) + str(s) + for secret in ("xoxb-secret-aaaaaaa", "xoxe.xoxp-secret-bbbbbbb", + "xoxe-1-secret-ccccccc"): + # A token is a whole-value credential: nothing of it survives, unlike the + # positional masking applied to a DSN's password. + assert secret not in text, f"{secret[:14]}... leaked into the settings repr" + assert s.aws_region in text, "control leg failed: the repr shows nothing at all" + + +def test_model_dump_is_not_used_on_settings_anywhere_in_src(): + """The redaction covers repr()/str() ONLY, by explicit design. + + `Settings.__repr_args__` masks credential-named fields, and its docstring records + that as closing "the only described leak path" for SEC-19 — deliberately leaving + fields as plain `str` rather than SecretStr to avoid churning ~130 call sites. + `model_dump()` therefore returns every secret in the clear. Measured: it does. + + That scoping is only safe while nothing dumps the settings object, so the invariant + that actually protects SEC-19 is this one, not a redaction test. If a future caller + needs `model_dump()`, the redaction has to be widened first. + + Adding the DSN redaction (`database_url`'s password) did not change this: it is a + `__repr_args__` rule, so `model_dump()` still returns that password in the clear + too. The measurement below pins the rationale instead of just asserting it. + + ``model_dump`` is not the only bulk-read: ``dict(settings)`` (pydantic v2 defines + ``__iter__``), ``vars(settings)`` and ``settings.__dict__`` each return every field + in the clear as well, and the original regex here — ``(settings|get_settings\\(\\)) + \\s*\\.model_dump`` — matched none of them. All five forms are MEASURED to leak + below before any of them is scanned for, so this is a check on the real leak set + rather than on one remembered member of it. The scan is AST-based because the + dangerous forms are `dict(x)`/`vars(x)` calls, which a line regex cannot bind to a + settings object. + """ + import ast + import pathlib + + leaky = Settings(_env_file=None, secret_key="dump-secret-value", + database_url="postgresql://u:dump-dsn-password@h/db") + + # Leg 1 — measure. Every bulk-read below must actually expose both secrets; a + # pydantic upgrade that redacted one of them would make scanning for it dead weight, + # and one that added a sixth form would show up here as a stale list. + dump = leaky.model_dump() + assert dump["secret_key"] == "dump-secret-value" + assert "dump-dsn-password" in dump["database_url"] + for label, rendered in ( + ("model_dump", str(dump)), + ("model_dump_json", leaky.model_dump_json()), + ("dict()", str(dict(leaky))), + ("vars()", str(vars(leaky))), + ("__dict__", str(leaky.__dict__)), + ): + assert "dump-secret-value" in rendered, f"{label} no longer leaks; update the scan" + assert "dump-dsn-password" in rendered, f"{label} no longer leaks the DSN password" + # Control for the measurement itself: repr/str DO redact, so "everything leaks" is + # not the trivially true statement it would be if __repr_args__ were broken. + assert "dump-secret-value" not in repr(leaky) + assert "dump-dsn-password" not in repr(leaky) + + # Leg 2 — scan. Names treated as a settings object: anything bound from + # get_settings(), plus the module-wide convention `settings` (21 files in src/ do + # `settings = get_settings()`), plus a `Settings(...)` construction. + LEAKY_ATTRS = ("model_dump", "model_dump_json", "__dict__") + LEAKY_BUILTINS = ("dict", "vars") + + def _is_settings_call(node): + if not isinstance(node, ast.Call): + return False + fn = node.func + name = fn.attr if isinstance(fn, ast.Attribute) else getattr(fn, "id", None) + return name in ("get_settings", "Settings") + + src = pathlib.Path(__file__).resolve().parents[2] / "src" + offenders = [] + for f in sorted(src.rglob("*.py")): + text = f.read_text() + tree = ast.parse(text, filename=str(f)) + names = {"settings"} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and _is_settings_call(node.value): + names.update(t.id for t in node.targets if isinstance(t, ast.Name)) + elif isinstance(node, ast.AnnAssign) and _is_settings_call(node.value): + if isinstance(node.target, ast.Name): + names.add(node.target.id) + for node in ast.walk(tree): + hit = None + if isinstance(node, ast.Attribute) and node.attr in LEAKY_ATTRS: + base = node.value + if (isinstance(base, ast.Name) and base.id in names) or _is_settings_call(base): + hit = f"{node.attr} on a settings object" + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in LEAKY_BUILTINS + and len(node.args) == 1 + and ( + (isinstance(node.args[0], ast.Name) and node.args[0].id in names) + or _is_settings_call(node.args[0]) + ) + ): + hit = f"{node.func.id}() over a settings object" + if hit: + line = text.splitlines()[node.lineno - 1].strip() + offenders.append(f"{f.relative_to(src.parent)}:{node.lineno}: {hit}: {line}") + assert not offenders, ( + "a bulk read of Settings returns unredacted secrets — only repr()/str() go " + "through __repr_args__. See Settings.__repr_args__; widen the redaction (or use " + "SecretStr) before adding these:\n" + "\n".join(offenders) + ) + # Control: the scan is actually looking at files. A glob that matched nothing would + # make the assertion above vacuous. + assert len(list(src.rglob("*.py"))) > 20 diff --git a/tests/unit/test_slack_web.py b/tests/unit/test_slack_web.py new file mode 100644 index 0000000..2ba1ff4 --- /dev/null +++ b/tests/unit/test_slack_web.py @@ -0,0 +1,224 @@ +"""Contract for the web-layer Slack boundary. + +Everything outside src/agent/slack_client.py goes through src/services/slack_web.py. +These tests pin the three properties the eight ex-call-sites were each missing: +full pagination, retry on 429, and splitting at 4000 characters. +""" +from unittest.mock import MagicMock + +import pytest +from slack_sdk.errors import SlackApiError + +from src.services import slack_web + + +def _resp(data): + r = MagicMock() + r.data = data + r.get = data.get + r.__getitem__ = lambda _s, k: data[k] + return r + + +def test_list_channel_ids_follows_every_cursor(monkeypatch): + pages = [ + {"channels": [{"name": "a", "id": "C1"}], + "response_metadata": {"next_cursor": "p2"}}, + {"channels": [{"name": "b", "id": "C2"}], + "response_metadata": {"next_cursor": ""}}, + ] + calls = [] + + client = MagicMock() + client.conversations_list.side_effect = lambda **kw: ( + calls.append(kw), _resp(pages[len(calls) - 1]))[1] + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert slack_web.list_channel_ids("xoxb-test") == {"a": "C1", "b": "C2"} + assert len(calls) == 2, "a single page is the defect this replaces" + assert calls[1]["cursor"] == "p2" + + +def test_lookup_user_by_email_retries_a_rate_limit(monkeypatch): + err = SlackApiError("ratelimited", _resp({"error": "ratelimited"})) + err.response.headers = {"Retry-After": "0"} + client = MagicMock() + client.users_lookupByEmail.side_effect = [ + err, _resp({"user": {"id": "U9"}}), + ] + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert slack_web.lookup_user_by_email("xoxb-test", "a@b.org") == "U9" + assert client.users_lookupByEmail.call_count == 2 + + +def test_lookup_user_by_email_returns_none_when_not_found(monkeypatch): + err = SlackApiError("users_not_found", _resp({"error": "users_not_found"})) + client = MagicMock() + client.users_lookupByEmail.side_effect = err + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert slack_web.lookup_user_by_email("xoxb-test", "nobody@b.org") is None + + +def test_get_user_info_returns_none_without_retrying(monkeypatch): + # users.info says `user_not_found`, users.lookupByEmail says `users_not_found`. + # Both are terminal: retrying costs the caller 3.5s of backoff in a synchronous + # request path to re-learn that a user who does not exist still does not. + err = SlackApiError("user_not_found", _resp({"error": "user_not_found"})) + client = MagicMock() + client.users_info.side_effect = err + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert slack_web.get_user_info("xoxb-test", "U404") is None + assert client.users_info.call_count == 1 + + +def test_post_message_splits_over_the_limit(monkeypatch): + client = MagicMock() + client.chat_postMessage.side_effect = lambda **kw: _resp({"ts": "1.0", "ok": True}) + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + posted = slack_web.post_message("xoxb-test", "#general", "x" * 9000) + + assert client.chat_postMessage.call_count >= 3 + for call in client.chat_postMessage.call_args_list: + assert len(call.kwargs["text"]) <= 4000 + assert len(posted) == client.chat_postMessage.call_count + + +def test_post_message_leaves_a_short_body_in_one_call(monkeypatch): + client = MagicMock() + client.chat_postMessage.side_effect = lambda **kw: _resp({"ts": "1.0", "ok": True}) + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert len(slack_web.post_message("xoxb-test", "#general", "short")) == 1 + assert client.chat_postMessage.call_count == 1 + + +def test_list_channel_ids_raises_rather_than_returning_a_subset(monkeypatch): + # An unrecognised error is retried, so page two has to fail on *every* attempt. + # A single error entry would exhaust side_effect and surface as StopIteration + # instead of the SlackListingIncomplete this test is about. + fatal = SlackApiError("fatal", _resp({"error": "fatal"})) + client = MagicMock() + client.conversations_list.side_effect = [ + _resp({"channels": [{"name": "a", "id": "C1"}], + "response_metadata": {"next_cursor": "p2"}}), + *[fatal] * slack_web._MAX_ATTEMPTS, + ] + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + # Retry delays are real sleeps; zero the base so the test costs nothing. + monkeypatch.setattr(slack_web, "_BACKOFF_BASE", 0) + + with pytest.raises(slack_web.SlackListingIncomplete) as exc: + slack_web.list_channel_ids("xoxb-test") + assert exc.value.partial == {"a": "C1"} + + +def test_post_message_threads_the_reply_when_thread_ts_is_given(monkeypatch): + """The two legacy PI-guidance callers post into a proposal thread. + + Without thread_ts they could not use this boundary at all: guidance posted + without one lands in the channel root instead of the thread, which is worse + than the raw client they used before. + """ + client = MagicMock() + client.chat_postMessage.side_effect = lambda **kw: _resp({"ts": "9.0", "ok": True}) + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + posted = slack_web.post_message( + "xoxb-test", "C123", "guidance", thread_ts="1700000000.000100") + + assert client.chat_postMessage.call_count == 1 + assert client.chat_postMessage.call_args.kwargs["thread_ts"] == "1700000000.000100" + assert posted[0]["thread_ts"] == "1700000000.000100" + + +def test_post_message_omits_thread_ts_entirely_when_not_threading(monkeypatch): + """A top-level post must be byte-identical to before thread_ts existed. + + Sending thread_ts=None would be a different Slack payload, so the key is + dropped rather than passed as None. + """ + client = MagicMock() + client.chat_postMessage.side_effect = lambda **kw: _resp({"ts": "9.0", "ok": True}) + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + slack_web.post_message("xoxb-test", "#general", "top level") + + assert "thread_ts" not in client.chat_postMessage.call_args.kwargs + + +# --------------------------------------------------------------------------- +# The async wrappers. Six of the seven call sites are FastAPI route handlers, and +# _call sleeps synchronously between retries, so calling the sync functions from +# an `async def` stalls the event loop for every request the process is serving — +# strictly worse than the raw WebClient they replaced, which had no retry at all. +# --------------------------------------------------------------------------- + + +async def test_the_async_wrapper_runs_the_blocking_call_off_the_event_loop(monkeypatch): + """The sync body must execute on a worker thread, not the loop's thread.""" + import threading + + loop_thread = threading.get_ident() + seen: dict[str, int] = {} + + def _record(**kw): + seen["thread"] = threading.get_ident() + return _resp({"user": {"id": "U1"}}) + + client = MagicMock() + client.users_lookupByEmail.side_effect = _record + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert await slack_web.lookup_user_by_email_async("xoxb-test", "a@b.org") == "U1" + assert seen["thread"] != loop_thread, ( + "the blocking Slack call ran on the event loop's own thread — one 429 " + "would freeze every other request in the process" + ) + + +async def test_every_sync_entry_point_has_an_async_wrapper(): + """A future call site must not have to choose the blocking variant by accident.""" + for name in ("list_channel_ids", "lookup_user_by_email", "get_user_info", + "join_channel", "post_message"): + assert hasattr(slack_web, f"{name}_async"), f"missing {name}_async" + assert f"{name}_async" in slack_web.__all__ + + +def test_an_outsized_retry_after_is_capped(monkeypatch): + """Slack can ask for a minute. Three of those would hold a request for minutes. + + The cap bounds request latency; it is only safe to sleep at all because the + async callers reach this through the _async wrappers. + """ + slept: list[float] = [] + monkeypatch.setattr(slack_web.time, "sleep", lambda d: slept.append(d)) + + err = SlackApiError("ratelimited", _resp({"error": "ratelimited"})) + err.response.headers = {"Retry-After": "600"} + client = MagicMock() + client.users_lookupByEmail.side_effect = [err, _resp({"user": {"id": "U2"}})] + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + assert slack_web.lookup_user_by_email("xoxb-test", "a@b.org") == "U2" + assert slept == [slack_web._MAX_RETRY_AFTER], ( + f"slept {slept} instead of capping at {slack_web._MAX_RETRY_AFTER}s" + ) + + +def test_a_modest_retry_after_is_honoured_exactly(monkeypatch): + """Under the cap, obey Slack — guessing is how a throttled bot gets blocked.""" + slept: list[float] = [] + monkeypatch.setattr(slack_web.time, "sleep", lambda d: slept.append(d)) + + err = SlackApiError("ratelimited", _resp({"error": "ratelimited"})) + err.response.headers = {"Retry-After": "7"} + client = MagicMock() + client.users_lookupByEmail.side_effect = [err, _resp({"user": {"id": "U3"}})] + monkeypatch.setattr(slack_web, "_client", lambda _t: client) + + slack_web.lookup_user_by_email("xoxb-test", "a@b.org") + assert slept == [7.0] diff --git a/tests/unit/test_thread_guidance.py b/tests/unit/test_thread_guidance.py new file mode 100644 index 0000000..6293f10 --- /dev/null +++ b/tests/unit/test_thread_guidance.py @@ -0,0 +1,62 @@ +import pytest + +from src.agent.thread_guidance import phase4_guidance + + +@pytest.mark.parametrize("count,expected", [(1, "EXPLORE"), (4, "EXPLORE"), + (5, "DECIDE"), (11, "DECIDE"), + (12, "MUST CONCLUDE"), (99, "MUST CONCLUDE")]) +def test_phase_boundaries_are_unchanged(count, expected): + for role in ("pi_lab", "scout_hub"): + assert phase4_guidance(role, count)[0] == expected + + +def test_pi_lab_strings_are_byte_identical_to_the_pinned_snapshot(): + # These exact strings are pinned in + # tests/characterization/__snapshots__/test_agent_turn_gm.ambr. Any drift here + # changes every PI bot's behaviour, which this refactor must not do. + _, guidance, instructions = phase4_guidance("pi_lab", 5) + assert guidance == ( + "You are in the DECIDE phase. Narrow the scope: is there genuine complementarity? " + "Can you name a specific first experiment? If yes, build toward a :memo: Summary proposal. " + "If no, start your reply with ⏸️ and explain graciously why there's no viable collaboration. " + "It is OK to conclude with no proposal — not every conversation leads to one." + ) + assert instructions == ( + "Write a reply that moves toward a conclusion. Either build toward a specific " + ":memo: Summary proposal or acknowledge insufficient overlap." + ) + + +def test_unknown_role_falls_back_to_pi_lab(): + assert phase4_guidance("nonexistent", 5) == phase4_guidance("pi_lab", 5) + + +def test_scout_hub_never_asks_for_a_collaboration_proposal(): + for count in (1, 5, 12): + _, guidance, instructions = phase4_guidance("scout_hub", count) + blob = guidance + instructions + assert ":memo:" not in blob + assert "collaboration proposal" not in blob + assert "your lab's recent work" not in blob + assert "complementarity" not in blob + + +def test_scout_hub_decide_phase_works_the_gating_criteria(): + _, guidance, instructions = phase4_guidance("scout_hub", 5) + blob = (guidance + instructions).lower() + assert "baltimore" in blob + assert "freedom-to-operate" in blob or "fto" in blob + assert "differentiation" in blob + # The measured failure: inferring the Baltimore gate from a JHU address. + assert "jhu address" in blob or "institution is not" in blob + # Part C.4 of the rubric — the target-level scientific checklist. + assert "proof of mechanism" in blob + + +def test_scout_hub_conclusion_carries_the_verdict_and_names_the_artifact(): + phase, guidance, instructions = phase4_guidance("scout_hub", 12) + assert phase == "MUST CONCLUDE" + blob = guidance + instructions + assert ":mag:" in blob + assert "⏸️" in blob diff --git a/tests/unit/test_tool_gating.py b/tests/unit/test_tool_gating.py new file mode 100644 index 0000000..1a73d32 --- /dev/null +++ b/tests/unit/test_tool_gating.py @@ -0,0 +1,26 @@ +import pytest + +from src.agent.tools import execute_tool, tools_for_role + + +def test_pi_lab_tool_list_excludes_hub_only_tools(): + names = {t["name"] for t in tools_for_role("pi_lab")} + assert "retrieve_profile" in names + assert "search_prior_art" not in names # true before Task 7; still true after + + +@pytest.mark.asyncio +async def test_executor_refuses_a_tool_not_in_the_role(): + # retrieve_foa is a pi_lab tool; ask a hypothetical role that lacks it. + # Use a role dir that does not exist -> DEFAULT_TOOLS (has retrieve_foa), + # so instead assert refusal via a role we can pin: monkeypatch load_role. + from src.agent import tools as tools_mod + from src.agent.roles import RoleSpec + + orig = tools_mod.load_role + tools_mod.load_role = lambda name: RoleSpec(name=name, label=name, tools=frozenset({"retrieve_profile"})) + try: + out = await execute_tool("retrieve_foa", {"foa_number": "PA-24-1"}, "su", None, role="locked") + finally: + tools_mod.load_role = orig + assert "not available" in out.lower() diff --git a/tests/unit/test_transport.py b/tests/unit/test_transport.py new file mode 100644 index 0000000..dfb1bb6 --- /dev/null +++ b/tests/unit/test_transport.py @@ -0,0 +1,238 @@ +"""Tests for the message transport abstraction (Slack-off mode). + +The second class covers the *outbound* half of the declared contract: `post_message` +reports one record per message the backend really created, and the engine writes one +`agent_messages` row per record. Both halves are needed and neither is observable from +inside our own database with `NullTransport`, which never splits — so a mirror that +recorded one row for a post the backend turned into five looked identical here (Rule S2) +and only showed up as messages in Slack with no row. +""" + +from src.agent.simulation import SimulationEngine +from src.agent.transport import NullTransport, Transport + + +class TestNullTransport: + def test_conforms_to_protocol(self): + t = NullTransport("su") + assert isinstance(t, Transport) + + def test_reports_disconnected(self): + t = NullTransport("su") + # is_connected=False makes the engine's guards take the no-op path. + assert t.is_connected is False + assert t.bot_user_id is None + assert t.connect() is True # usable, just not Slack-backed + + def test_outbound_posts_are_noops(self): + t = NullTransport("su") + # No Slack ts — the engine mints a local canonical id instead. + assert t.post_message("general", "hi") is None + assert t.send_dm("U1", "hi") is None + assert t.open_dm_channel("U1") is None + + def test_channel_creates_use_local_ids(self): + t = NullTransport("su") + assert t.create_channel("general") == {"id": "local:general", "name": "general"} + priv = t.create_private_channel("priv-a-b") + assert priv["id"] == "local:priv-a-b" + assert priv["is_private"] is True + assert t.invite_to_channel("local:x", ["U1", "U2"]) is True + assert t.join_channel("local:x") is None + + def test_inbound_polls_return_empty(self): + t = NullTransport("su") + assert t.poll_channel_messages("local:general") == [] + assert t.get_thread_replies("local:general", "1.0") == [] + assert t.get_full_channel_history("local:general") == [] + assert t.get_all_thread_replies("local:general", "1.0") == [] + assert t.poll_dm_messages("U1") == [] + + def test_slack_client_conforms_to_protocol(self): + # The real client must structurally satisfy the same Protocol. + from src.agent.slack_client import AgentSlackClient + client = AgentSlackClient(agent_id="su", bot_token="xoxb-test") + assert isinstance(client, Transport) + + def test_cache_channel_ids_seeds_the_lookup(self): + t = NullTransport("su") + t.cache_channel_ids({"general": "local:general", "funding": "C123"}) + assert t.get_channel_id("general") == "local:general" + assert t.list_channels()["funding"] == "C123" + + def test_dead_visibility_setter_removed(self): + # set_visibility_lookup had 0 callers — dead code, dropped (M2). + assert not hasattr(NullTransport("su"), "set_visibility_lookup") + from src.agent.slack_client import AgentSlackClient + assert not hasattr(AgentSlackClient, "set_visibility_lookup") + + +class TestChannelCacheContract: + """M2: the channel name→id write path is part of the declared Transport + contract, so the engine seeds it via a public method rather than poking a + private ``_channel_name_to_id`` attribute that a new backend need not have.""" + + def test_cache_channel_ids_is_declared_on_the_protocol(self): + assert hasattr(Transport, "cache_channel_ids") + from src.agent.slack_client import AgentSlackClient + assert hasattr(AgentSlackClient, "cache_channel_ids") + + def test_backend_without_private_attr_can_be_seeded(self): + # Before M2 the engine crashed with AttributeError on a Protocol-only + # backend. Now the seed goes through cache_channel_ids, which any + # conforming backend implements however it likes. + class StubBackend: + def __init__(self): + self._seeded: dict[str, str] = {} + + def cache_channel_ids(self, mapping: dict[str, str]) -> None: + self._seeded.update(mapping) + + def get_channel_id(self, name: str) -> str | None: + return self._seeded.get(name) + + b = StubBackend() + assert not hasattr(b, "_channel_name_to_id") + b.cache_channel_ids({"general": "C1"}) + assert b.get_channel_id("general") == "C1" + + +class _SplittingTransport: + """A backend that reports what it really posted, per the declared contract. + + Deliberately not `FakeSlackClient`: that one always answers with a single ts, so it + cannot express "this text became three messages" — which is the case that lost four + rows out of five in production. + """ + + def __init__(self, chunks_per_post: int = 1): + self.agent_id = "su" + self.chunks_per_post = chunks_per_post + self.calls: list[dict] = [] + self._n = 1_700_000_000 + + def connect(self) -> bool: + return True + + @property + def is_connected(self) -> bool: + return True + + @property + def bot_user_id(self) -> str | None: + return "U_SU" + + def post_message(self, channel, text, thread_ts=None): + self.calls.append({"channel": channel, "text": text, "thread_ts": thread_ts}) + posted = [] + for index in range(self.chunks_per_post): + self._n += 1 + ts = f"{self._n}.000000" + parent = thread_ts if (thread_ts or index == 0) else posted[0]["ts"] + posted.append({ + "ts": ts, "channel": f"C_{channel}", + "text": f"{text}#{index}", "thread_ts": parent, + }) + return {**posted[0], "posted_messages": posted} + + def _resolve_channel_id(self, channel): + return channel if channel.startswith(("C", "G")) else f"C_{channel}" + + +class TestPostResultContract: + """`post_message` -> `posted_messages` -> one row per message, in bijection.""" + + def _engine(self, transport): + from src.agent.agent import Agent + + return SimulationEngine(agents=[Agent("su", "SuBot", "Andrew Su")], + slack_clients={"su": transport}) + + # --- the normaliser, in isolation ------------------------------------------- + + def test_nothing_posted_reports_no_messages(self): + """Which is the signal to mint a local canonical id instead.""" + assert SimulationEngine._mirrored_messages(None, "text", None) == [] + + def test_a_backend_that_never_splits_may_omit_the_key(self): + """`NullTransport` and any simple backend report a bare result; it describes the + one message it made, and the source text is what that message carries.""" + out = SimulationEngine._mirrored_messages( + {"ts": "1.0", "channel": "C_X"}, "hello", "0.5", + ) + assert out == [{"ts": "1.0", "channel": "C_X", "text": "hello", "thread_ts": "0.5"}] + + def test_reported_messages_are_passed_through_unchanged(self): + posted = [{"ts": "1.0", "channel": "C_X", "text": "a", "thread_ts": None}, + {"ts": "2.0", "channel": "C_X", "text": "b", "thread_ts": "1.0"}] + assert SimulationEngine._mirrored_messages( + {"ts": "1.0", "posted_messages": posted}, "a b", None, + ) == posted + + # --- and what the engine does with them ------------------------------------- + + async def test_a_split_post_becomes_one_row_per_real_message(self): + """Recording a single row for a post the backend turned into three left two of + them in Slack with no row at all, named the row's `slack_ts` after the *tail*, + and made the next restart's Slack reconcile ingest the unrecorded head chunks as + brand-new inbound messages. + """ + t = _SplittingTransport(chunks_per_post=3) + engine = self._engine(t) + await engine._post_message("su", "general", "a very long post") + + rows = list(engine.message_log._entries) + assert len(rows) == 3, [r.content for r in rows] + assert [r.ts for r in rows] == [r.slack_ts for r in rows], ( + "a mirrored row must record the backend's ts as its canonical id" + ) + # Each row carries the text of its own message, not the whole post three times. + assert [r.content for r in rows] == [ + "a very long post#0", "a very long post#1", "a very long post#2", + ] + # One logical post stays ONE top-level post: the continuations hang off chunk 0, + # so nobody else's Phase 2 scan sees three roots where one post was written. + assert rows[0].thread_ts is None + assert [r.thread_ts for r in rows[1:]] == [rows[0].ts, rows[0].ts] + assert [r.slack_thread_ts for r in rows[1:]] == [rows[0].ts, rows[0].ts] + assert len({r.ts for r in rows}) == 3, "two rows share a canonical id" + + async def test_an_unsplit_post_becomes_exactly_one_row(self): + """Control: an engine that always wrote three rows would pass the test above.""" + engine = self._engine(_SplittingTransport(chunks_per_post=1)) + await engine._post_message("su", "general", "one message") + rows = list(engine.message_log._entries) + assert len(rows) == 1 + assert rows[0].content == "one message#0" + assert rows[0].thread_ts is None + + async def test_every_chunk_of_a_split_reply_keeps_the_callers_thread(self): + """Control for the root case: a reply's continuations belong to the thread the + caller named, not to a sub-thread on the first chunk.""" + t = _SplittingTransport(chunks_per_post=3) + engine = self._engine(t) + from src.agent.message_log import LogEntry + engine.message_log.append(LogEntry( + ts="1700000000.000000", channel="general", sender_agent_id="su", + sender_name="SuBot", content="root", posted_at=1700000000.0, is_bot=True, + slack_ts="1700000000.000000", + )) + + await engine._post_message("su", "general", "long reply", + thread_ts="1700000000.000000") + + replies = [e for e in engine.message_log._entries if e.content.startswith("long reply")] + assert len(replies) == 3 + assert {r.thread_ts for r in replies} == {"1700000000.000000"} + assert {r.slack_thread_ts for r in replies} == {"1700000000.000000"} + + async def test_an_unmirrored_post_still_records_exactly_one_row(self): + """Slack-off: nothing was posted, so the engine mints one canonical id. Iterating + an empty report must not skip the row entirely.""" + engine = SimulationEngine(agents=[], slack_clients={}) + await engine._post_message("su", "general", "written with slack off") + rows = list(engine.message_log._entries) + assert len(rows) == 1 + assert rows[0].slack_ts is None and rows[0].slack_channel_id is None + assert rows[0].content == "written with slack off" + assert float(rows[0].ts) > 0