org1 parity: generic blackbird work + prod hardening, agent behaviour frozen (0018→0024)

.gitignore

@@ -66,6 +66,11 @@ data/agent_roster.json
 # Local scratch notes
 .notes/
 
+# Superpowers working files (implementation plans, per-session scratch). Local
+# process artifacts, not project documentation — docs/agent-flow.md and specs/
+# are where anything durable belongs.
+docs/superpowers/
+
 # Slack provisioning state — holds app client_secrets during a bulk provisioning
 # run (scripts/provision_slack_bots.py). Must never be committed.
 .provision_state.json
@@ -75,6 +80,12 @@ data/agent_roster.json
 .mutmut-cache
 mutants/
 
+# Playwright MCP browser artifacts (console logs, page snapshots)
+.playwright-mcp/
+
+# Production migration dumps (scripts/migrate/run_migration.sh). Never commit these:
+# they are full database copies containing message bodies and tokens.
+backups/
 # Local Claude Code state
 .claude/
 

CLAUDE.md

@@ -2,9 +2,26 @@
 
 ## Testing
 
-Run `python -m pytest tests/ -v` before committing. All tests must pass.
-Tests run inside Docker: `docker compose exec app python -m pytest tests/ -v`
-(may need `pip install pytest pytest-asyncio` first if the container was rebuilt).
+Run `./scripts/ci.sh` before committing — alembic sanity (single head, no
+duplicate revision ids), `ruff check` on the test suite, then the full pytest run
+with a branch-coverage floor. This is exactly what the `pre-push` hook runs, and
+it is the whole gate: there is no server-side CI.
+
+To run pytest alone **inside the container**, `TEST_DATABASE_URL` is required.
+Without it `tests/conftest.py` falls back to spinning an ephemeral Postgres via
+testcontainers, and the `app` container has no Docker socket — so every test that
+needs a database errors out (469 of them, measured 2026-08-04):
+
+```bash
+docker compose exec -T -e TEST_DATABASE_URL=postgresql+asyncpg://copi:copi@postgres:5432/copi_a3 \
+  app python -m pytest tests/ -v
+```
+
+The named database must already exist — the suite migrates it, it does not create
+it. Add a fresh scratch DB with
+`docker compose exec -T postgres createdb -U copi copi_xN`, and give concurrent
+suites distinct names so they do not migrate each other's schema mid-run. Never
+point `TEST_DATABASE_URL` at `copi`, the dev database.
 
 ## Compose file set (read this before any `docker compose` command)
 
@@ -68,8 +85,12 @@ C="-f docker-compose.prod.yml -f docker-compose.override.yml"
 docker logs agent-run > logs/run_$(date +%s).log 2>&1
 ls -t logs/run_*.log | tail -n +11 | xargs rm -f
 
-# 2. Stop the old container
-docker rm -f agent-run
+# 2. Stop the old container — GRACEFULLY. `docker rm -f` sends SIGKILL, which
+#    skips the shutdown flush and permanently loses the in-flight turn's
+#    messages (the DB, not Slack, is the durable store). `docker stop` sends
+#    SIGTERM; -t 30 leaves room for an in-flight LLM call to finish.
+docker stop -t 30 agent-run
+docker rm agent-run
 
 # 3. Rebuild app + worker (picks up code changes)
 docker compose $C up -d --build app worker
@@ -135,14 +156,14 @@ rotating pair is persisted in the `app_settings` KV table) and a public `base_ur
 roster from the container, then run the script on the host:
 
 ```bash
-docker exec copi-python-app-1 python scripts/export_agent_roster.py   # writes data/agent_roster.json
+docker compose exec app python scripts/export_agent_roster.py   # writes data/agent_roster.json
 python3 scripts/provision_slack_bots.py                               # host: creates apps, prints OAuth URLs
 ```
 
 The host script writes tokens to `.env`; import them into the DB column with:
 
 ```bash
-docker exec copi-python-app-1 python scripts/backfill_agent_tokens.py
+docker compose exec app python scripts/backfill_agent_tokens.py
 ```
 
 (`.env` + `config.py get_slack_tokens()` remain a read fallback, but the DB column is

README.md

@@ -37,7 +37,14 @@ Cross-cutting:
 ```bash
 cp .env.example .env   # fill in Anthropic, Slack, ORCID, SMTP credentials
 docker compose up -d --build app worker postgres
+
+# Migrate. Check for a single head FIRST: two migrations sharing a revision id
+# (a stale branch renumbered late) makes `upgrade head` fail on multiple heads,
+# and makes a targeted `upgrade <rev>` silently skip one of them while stamping
+# the DB as fully migrated. `alembic heads` needs no database.
+docker compose exec app alembic heads      # must print exactly one line
 docker compose exec app alembic upgrade head
+docker compose exec app alembic current    # confirm it advanced
 ```
 
 Web UI: <http://localhost:8001>.
@@ -71,7 +78,8 @@ Before restarting, save logs and rebuild:
 ```bash
 docker logs agent-run > logs/run_$(date +%s).log 2>&1
 ls -t logs/run_*.log | tail -n +11 | xargs rm -f
-docker rm -f agent-run
+docker stop -t 30 agent-run   # SIGTERM: lets the engine flush before exit
+docker rm agent-run
 docker compose up -d --build app worker
 docker compose --profile agent run -d --name agent-run agent \
   python -m src.agent.main --budget 0

alembic/env.py

@@ -42,6 +42,39 @@ def run_migrations_offline() -> None:
         context.run_migrations()
 
 
+#: How long a migration will WAIT for a lock before giving up, in milliseconds.
+#: 0 disables the bound (Postgres' own default, and what this file did before).
+#:
+#: Why this exists. `context.configure()` below is deliberately NOT passed
+#: `transaction_per_migration`, so the entire upgrade chain runs in ONE
+#: transaction. That is good — a killed migration cannot leave a half-applied
+#: schema, verified by terminating the backend mid-chain. But it means every lock
+#: the chain takes is held until the final commit, and migration 0019 takes
+#: ACCESS EXCLUSIVE on `agent_messages` to add three indexes and a unique
+#: constraint (which brings a fourth index of its own).
+#:
+#: With no lock_timeout, `alembic upgrade` parked behind a single open
+#: `BEGIN; SELECT …` waits forever — and because a pending ACCESS EXCLUSIVE
+#: request queues ahead of new readers, every subsequent query on that table
+#: blocks behind it. One forgotten transaction plus a migration is a total stall
+#: on the hot table, with no timeout to end it. Failing fast and retrying in a
+#: quieter moment is strictly better than an unbounded outage: the transaction
+#: rolls back cleanly, so a timeout costs nothing but the attempt.
+#:
+#: This bounds only the WAIT for a lock. It is not `statement_timeout`, which
+#: would cancel a legitimately long index build partway through.
+LOCK_TIMEOUT_MS = os.environ.get("ALEMBIC_LOCK_TIMEOUT_MS", "10000")
+
+
+#: NOTE ON HOW THIS IS APPLIED. It is set as an asyncpg *connect* setting, not by
+#: executing `SET lock_timeout = …` on the connection inside do_run_migrations().
+#: The obvious version of that is quietly catastrophic: `connection.exec_driver_sql`
+#: before `context.begin_transaction()` opens its own transaction, alembic's
+#: transaction then nests inside it, and the outer `async with connect()` exits
+#: without committing — so every migration LOGS "Running upgrade" and the whole
+#: chain SILENTLY ROLLS BACK, leaving no `alembic_version` table at all. Observed
+#: while writing this: 18 migrations "applied", nothing persisted. Counting the log
+#: lines is not a verification; always re-read `alembic_version` afterwards.
 def do_run_migrations(connection: Connection) -> None:
     context.configure(connection=connection, target_metadata=target_metadata)
     with context.begin_transaction():
@@ -50,11 +83,15 @@ def do_run_migrations(connection: Connection) -> None:
 
 async def run_async_migrations() -> None:
     """In this scenario we need to create an Engine and associate a connection with the context."""
-    connectable = async_engine_from_config(
-        config.get_section(config.config_ini_section, {}),
-        prefix="sqlalchemy.",
-        poolclass=pool.NullPool,
-    )
+    cfg = config.get_section(config.config_ini_section, {})
+    kwargs: dict = {"prefix": "sqlalchemy.", "poolclass": pool.NullPool}
+    if LOCK_TIMEOUT_MS and LOCK_TIMEOUT_MS != "0":
+        # asyncpg takes libpq-style GUCs via server_settings, applied at connect
+        # time — outside any transaction, so it cannot disturb alembic's.
+        kwargs["connect_args"] = {
+            "server_settings": {"lock_timeout": str(int(LOCK_TIMEOUT_MS))}
+        }
+    connectable = async_engine_from_config(cfg, **kwargs)
     async with connectable.connect() as connection:
         await connection.run_sync(do_run_migrations)
     await connectable.dispose()

alembic/versions/0019_agent_message_content.py

@@ -0,0 +1,85 @@
+"""Add conversation-content columns to agent_messages (DB becomes primary store)
+
+Revision ID: 0019
+Revises: 0018
+Create Date: 2026-07-20 00:00:00.000000
+
+Makes the local DB the primary store for agent conversations: agent_messages now
+carries the message body and sender metadata (previously only in Slack + the
+in-memory MessageLog), plus nullable Slack-mirror mapping columns. See
+specs/local-db-conversations.md.
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "0019"
+down_revision: Union[str, None] = "0018"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Content columns — DB is now the durable conversation store.
+    op.add_column(
+        "agent_messages",
+        sa.Column("content", sa.Text(), nullable=False, server_default=""),
+    )
+    op.add_column(
+        "agent_messages",
+        sa.Column("sender_name", sa.String(100), nullable=False, server_default=""),
+    )
+    op.add_column(
+        "agent_messages",
+        sa.Column("is_bot", sa.Boolean(), nullable=False, server_default=sa.text("true")),
+    )
+    op.add_column(
+        "agent_messages",
+        sa.Column("posted_at", sa.Float(), nullable=False, server_default="0"),
+    )
+    # Slack-mirror mapping (NULL when Slack is off / message is DB-origin).
+    op.add_column("agent_messages", sa.Column("slack_ts", sa.String(50), nullable=True))
+    op.add_column("agent_messages", sa.Column("slack_channel_id", sa.String(100), nullable=True))
+    op.add_column("agent_messages", sa.Column("slack_thread_ts", sa.String(50), nullable=True))
+
+    # agent_id becomes the sender_agent_id: NULL for human/PI messages.
+    op.alter_column("agent_messages", "agent_id", existing_type=sa.String(50), nullable=True)
+
+    # Idempotency + rebuild/mirror indexes.
+    op.create_unique_constraint(
+        "uq_agent_messages_run_ts", "agent_messages", ["simulation_run_id", "message_ts"]
+    )
+    op.create_index(
+        "ix_agent_messages_run_posted",
+        "agent_messages",
+        ["simulation_run_id", "posted_at"],
+    )
+    op.create_index(
+        "ix_agent_messages_run_channel_posted",
+        "agent_messages",
+        ["simulation_run_id", "channel_name", "posted_at"],
+    )
+    op.create_index(
+        "ix_agent_messages_run_slack_ts",
+        "agent_messages",
+        ["simulation_run_id", "slack_ts"],
+        postgresql_where=sa.text("slack_ts IS NOT NULL"),
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_agent_messages_run_slack_ts", table_name="agent_messages")
+    op.drop_index("ix_agent_messages_run_channel_posted", table_name="agent_messages")
+    op.drop_index("ix_agent_messages_run_posted", table_name="agent_messages")
+    op.drop_constraint("uq_agent_messages_run_ts", "agent_messages", type_="unique")
+    op.alter_column("agent_messages", "agent_id", existing_type=sa.String(50), nullable=False)
+    op.drop_column("agent_messages", "slack_thread_ts")
+    op.drop_column("agent_messages", "slack_channel_id")
+    op.drop_column("agent_messages", "slack_ts")
+    op.drop_column("agent_messages", "posted_at")
+    op.drop_column("agent_messages", "is_bot")
+    op.drop_column("agent_messages", "sender_name")
+    op.drop_column("agent_messages", "content")

alembic/versions/0020_pi_dm_messages.py

@@ -0,0 +1,66 @@
+"""Add pi_dm_messages table (durable PI<->bot direct messages)
+
+Revision ID: 0020
+Revises: 0019
+Create Date: 2026-07-20 00:00:00.000000
+
+DMs never entered the shared message log, so they had no durable home. This
+table stores them so a PI can DM their bot (standing instructions, questions)
+with Slack fully off. See specs/local-db-conversations.md.
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+revision: str = "0020"
+down_revision: Union[str, None] = "0019"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "pi_dm_messages",
+        sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
+        sa.Column(
+            "simulation_run_id",
+            postgresql.UUID(as_uuid=True),
+            sa.ForeignKey("simulation_runs.id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("agent_id", sa.String(50), nullable=False),
+        sa.Column("pi_user_id", sa.String(50), nullable=False),
+        sa.Column(
+            "direction",
+            sa.Enum("inbound", "outbound", name="pi_dm_direction_enum"),
+            nullable=False,
+        ),
+        sa.Column("content", sa.Text(), nullable=False),
+        sa.Column("sender_name", sa.String(100), nullable=False, server_default=""),
+        sa.Column("ts", sa.String(50), nullable=False),
+        sa.Column("slack_ts", sa.String(50), nullable=True),
+        sa.Column("posted_at", sa.Float(), nullable=False, server_default="0"),
+        sa.Column(
+            "created_at", sa.DateTime(timezone=True),
+            server_default=sa.func.now(), nullable=False,
+        ),
+    )
+    op.create_index(
+        "ix_pi_dm_run_agent_posted", "pi_dm_messages",
+        ["simulation_run_id", "agent_id", "posted_at"],
+    )
+    op.create_index(
+        "ix_pi_dm_run_direction_posted", "pi_dm_messages",
+        ["simulation_run_id", "direction", "posted_at"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_pi_dm_run_direction_posted", table_name="pi_dm_messages")
+    op.drop_index("ix_pi_dm_run_agent_posted", table_name="pi_dm_messages")
+    op.drop_table("pi_dm_messages")
+    sa.Enum(name="pi_dm_direction_enum").drop(op.get_bind(), checkfirst=True)

alembic/versions/0021_inbox_cursor_created_at_indexes.py

@@ -0,0 +1,39 @@
+"""Index the DB inbox pollers' created_at cursor
+
+Revision ID: 0021
+Revises: 0020
+Create Date: 2026-07-25 00:00:00.000000
+
+Both DB inbox pollers used to page over ``posted_at``, which is derived from the
+*writing process's* clock (float of its minted ts). That made inbound PI delivery
+depend on every writer's clock agreeing with the engine's to within the lookback
+window — fine on one host, silently lossy across hosts. They now page over
+``created_at`` (``server_default=now()``, i.e. the single Postgres server's
+clock), so these indexes back the new access path. See
+.notes/db-conversations-residual-2026-07-24.md (R3).
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "0021"
+down_revision: Union[str, None] = "0020"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_index(
+        "ix_agent_messages_run_created", "agent_messages",
+        ["simulation_run_id", "created_at"],
+    )
+    op.create_index(
+        "ix_pi_dm_run_direction_created", "pi_dm_messages",
+        ["simulation_run_id", "direction", "created_at"],
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_pi_dm_run_direction_created", table_name="pi_dm_messages")
+    op.drop_index("ix_agent_messages_run_created", table_name="agent_messages")