Serve the feed timeline from D1; Fly proxy becomes an ingest-push crawler

.github/workflows/feed-proxy-deploy.yml

@@ -54,6 +54,43 @@ jobs:
       - name: Run tests
         run: bun test
 
+  deploy-staging:
+    name: Deploy to Staging
+    needs: [typecheck, test]
+    # Same cadence as the staging Worker: every push to main. Staging has its own
+    # Fly app (fly.staging.toml) so it never shares the prod machine or its cache,
+    # and config drift between the two files surfaces here before a release.
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - name: Setup flyctl
+        uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # master
+
+      # The app is provisioned by hand (Phase 3 of docs/plans/D1_FEED_TIMELINE.md:
+      # app + volume + PROXY_SECRET). Until that happens this job would fail on
+      # every push to main, so skip it with a notice instead.
+      - name: Check staging app exists
+        id: staging_app
+        run: |
+          if flyctl status -a skyreader-feed-proxy-staging >/dev/null 2>&1; then
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::skyreader-feed-proxy-staging not provisioned yet; skipping staging deploy (see docs/plans/D1_FEED_TIMELINE.md, Phase 3)."
+          fi
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+
+      - name: Deploy
+        if: steps.staging_app.outputs.exists == 'true'
+        run: flyctl deploy --remote-only --config fly.staging.toml
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+
   deploy:
     name: Deploy to Production
     needs: [typecheck, test]

CLAUDE.md

@@ -9,7 +9,7 @@ This is a monorepo with 6 packages:
 - `backend/` - Cloudflare Workers API
 - `frontend/` - SvelteKit PWA
 - `admin/` - SvelteKit admin dashboard (Cloudflare Pages)
-- `feed-proxy/` - Feed caching proxy (Fly.io)
+- `feed-proxy/` - Feed crawler + article extraction (Fly.io; one app per environment)
 - `linkblog-site/` - Standalone SvelteKit app rendering public linkblogs at `linkblogs.skyreader.app` (Cloudflare Pages)
 - `extension/` - Chrome extension for one-click saves with live-DOM article extraction (Manifest V3)
 
@@ -85,6 +85,9 @@ This script runs D1 migrations, then starts the feed proxy (port 3000), backend
   ```
   FRONTEND_URL=http://127.0.0.1:5173
   FEED_PROXY_URL=http://127.0.0.1:3000
+  # Shared with the proxy (dev-local.sh exports the same value as PROXY_SECRET).
+  # Feed ingest is fail-closed, so without this the local reader stays empty.
+  FEED_PROXY_SECRET=dev-proxy-secret
   ```
 
 **Resetting the local database:**
@@ -217,14 +220,19 @@ AT Protocol (Bluesky PDS) + Fly.io Feed Proxy + Jetstream Firehose
 - **Database:** D1 (SQLite) - reads the same database as the backend
 - **Features:** Ops panel (cron liveness, firehose lag, proxy cache health) with 30-day trend
   sparklines, system metrics, user management, feed health monitoring, search/sort/pagination
-- **Pages:** Dashboard (ops + metrics + trends), Users (list + detail), Feeds (health + error tracking)
+- **Pages:** Dashboard (ops + metrics + trends), Users (list + detail), Feeds (health + error tracking).
+  Feed health is the crawler's own verdict from `feeds.error_count` / `feeds.crawl_stale`, not an
+  inference from `last_ingest_at` — that only moves when a feed publishes, so it says nothing about
+  whether the feed still works.
 - **Deploy:** Cloudflare Pages via GitHub Actions (staging on push to main, production on release)
 
 ### Key Data Flow
 
 1. **Auth:** Handle → DID resolution → OAuth PKCE/DPoP → session token
 2. **Subscriptions:** Stored in user's PDS, cached locally in IndexedDB
-3. **Feed Updates:** Frontend requests feeds → backend proxies via Fly.io feed cache
+3. **Feed Updates:** Fly.io proxy crawls feeds and pushes new items into D1 (`feed_items`); the
+   frontend refreshes with one `GET /api/v2/timeline` query that joins subscriptions and read
+   state. Reads never touch Fly — see `docs/plans/D1_FEED_TIMELINE.md`
 4. **Social:** Jetstream firehose → D1 shares table → frontend polls for updates
 
 ## AT Protocol Integration

README.md

@@ -40,6 +40,9 @@ cd skyreader
   ```
   FRONTEND_URL=http://127.0.0.1:5173
   FEED_PROXY_URL=http://127.0.0.1:3000
+  # Shared with the proxy (dev-local.sh exports the same value as PROXY_SECRET).
+  # Feed ingest is fail-closed, so without this the local reader stays empty.
+  FEED_PROXY_SECRET=dev-proxy-secret
   ```
 
 ### Local Development

admin/src/lib/components/StatusBadge.svelte

@@ -1,9 +1,13 @@
 <script lang="ts">
   interface Props {
     status: 'healthy' | 'warning' | 'error';
+    // Overrides the generic severity word. Worth setting wherever the severity
+    // alone is ambiguous — "Erroring" and "Not crawled" are different faults that
+    // would both read as "Error"/"Warning".
+    label?: string;
   }
 
-  let { status }: Props = $props();
+  let { status, label }: Props = $props();
 
   const labels: Record<string, string> = {
     healthy: 'Healthy',
@@ -12,7 +16,7 @@
   };
 </script>
 
-<span class="badge {status}">{labels[status]}</span>
+<span class="badge {status}">{label ?? labels[status]}</span>
 
 <style>
   .badge {

admin/src/lib/metrics/feeds.test.ts

@@ -0,0 +1,33 @@
+import { describe, it, expect } from 'vitest';
+import { feedHealth } from './feeds';
+
+describe('feedHealth', () => {
+  it('reports a feed the crawler cannot fetch as an error', () => {
+    expect(feedHealth({ error_count: 4, crawl_stale: 0 })).toEqual({
+      status: 'error',
+      label: 'Erroring',
+    });
+  });
+
+  it('reports a feed the crawler never reaches as a distinct warning', () => {
+    // Not the same fault as erroring: nothing is failing, the crawler simply
+    // isn't keeping up. One is fixed per feed, the other by adding capacity.
+    expect(feedHealth({ error_count: 0, crawl_stale: 1 })).toEqual({
+      status: 'warning',
+      label: 'Not crawled',
+    });
+  });
+
+  it('leads with the error when a feed is both erroring and starved', () => {
+    expect(feedHealth({ error_count: 2, crawl_stale: 1 }).label).toBe('Erroring');
+  });
+
+  it('calls a quiet feed healthy', () => {
+    // The regression this replaced: a feed that simply hasn't published is fine,
+    // and `last_ingest_at` deliberately plays no part in the verdict.
+    expect(feedHealth({ error_count: 0, crawl_stale: 0 })).toEqual({
+      status: 'healthy',
+      label: 'OK',
+    });
+  });
+});

admin/src/lib/metrics/feeds.ts

@@ -1,26 +1,142 @@
-import type { MetricDefinition } from '$lib/types';
+import type { FeedRow, MetricDefinition } from '$lib/types';
+
+export interface FeedHealthVerdict {
+  status: 'healthy' | 'warning' | 'error';
+  label: string;
+}
+
+/**
+ * How one feed's row reads on the Feeds page.
+ *
+ * The crawler's verdict, not an inference. Health used to be derived from
+ * `last_ingest_at`, which only moves when a fetch produces a NEW item — so every
+ * feed that simply hadn't published in an hour showed as broken, and the page's
+ * status column carried no information. The two faults are also genuinely
+ * different: erroring means the fetch fails (actionable per feed), starved means
+ * the crawler never gets to it (actionable on capacity), so they don't collapse
+ * into one severity.
+ */
+export function feedHealth(feed: Pick<FeedRow, 'error_count' | 'crawl_stale'>): FeedHealthVerdict {
+  if (feed.error_count > 0) return { status: 'error', label: 'Erroring' };
+  if (feed.crawl_stale) return { status: 'warning', label: 'Not crawled' };
+  return { status: 'healthy', label: 'OK' };
+}
+
+// D1's hard ceiling is 10 GB. Alert well before it, so there's time to act
+// (lower the ingest content cap → tier old bodies to R2 → revisit retention).
+const ARCHIVE_ALERT_BYTES = 6 * 1024 * 1024 * 1024;
+
+// The per-feed sanity cap is 5,000 items (backend/src/routes/ingest.ts). A feed
+// approaching it is a GUID-churn bug signal, not steady state.
+const CHURN_WARN_ITEMS = 3000;
 
 export const feedMetrics: MetricDefinition[] = [
   {
     id: 'total_feeds',
     category: 'Feeds',
+    query: async (db) => {
+      const r = await db.prepare('SELECT COUNT(*) as count FROM feeds').first<{ count: number }>();
+      return { label: 'Crawled Feeds', value: r?.count ?? 0 };
+    },
+  },
+  {
+    id: 'erroring_feeds',
+    category: 'Feeds',
+    // Feeds a real user subscribes to that the crawler cannot fetch. This
+    // replaces the old "Subscribed Feeds Not Ingesting", which keyed off
+    // `last_ingest_at` and therefore counted every feed that simply hadn't
+    // published in an hour — it warned permanently and told an operator nothing.
+    // The crawler's own error verdict is the actionable number.
     query: async (db) => {
       const r = await db
-        .prepare('SELECT COUNT(*) as count FROM feed_metadata')
+        .prepare(
+          `SELECT COUNT(*) as count FROM feeds f
+            WHERE f.error_count > 0
+              AND EXISTS (SELECT 1 FROM subscriptions_cache sc
+                           WHERE sc.feed_url = f.feed_url AND sc.active = 1)`
+        )
         .first<{ count: number }>();
-      return { label: 'Total Feeds', value: r?.count ?? 0 };
+      const count = r?.count ?? 0;
+      return {
+        label: 'Subscribed Feeds Erroring',
+        value: count,
+        status: count > 0 ? 'warning' : 'healthy',
+      };
+    },
+  },
+  {
+    id: 'starved_feeds',
+    category: 'Feeds',
+    // In the crawl set, not erroring, and still not fetched for hours: the
+    // crawler is not keeping up. This is the failure the warm-loop batch cap
+    // produces, and the one the old stale-ingest metric was reaching for.
+    query: async (db) => {
+      const r = await db
+        .prepare(
+          `SELECT COUNT(*) as count FROM feeds f
+            WHERE f.crawl_stale = 1 AND f.error_count = 0
+              AND EXISTS (SELECT 1 FROM subscriptions_cache sc
+                           WHERE sc.feed_url = f.feed_url AND sc.active = 1)`
+        )
+        .first<{ count: number }>();
+      const count = r?.count ?? 0;
+      return {
+        label: 'Subscribed Feeds Not Being Crawled',
+        value: count,
+        status: count > 0 ? 'error' : 'healthy',
+      };
+    },
+  },
+  {
+    id: 'archive_items',
+    category: 'Feeds',
+    query: async (db) => {
+      const r = await db
+        .prepare('SELECT COUNT(*) as count FROM feed_items')
+        .first<{ count: number }>();
+      return { label: 'Archived Items', value: r?.count ?? 0 };
+    },
+  },
+  {
+    id: 'archive_size',
+    category: 'Feeds',
+    query: async (db) => {
+      // Estimated, not exact: summing LENGTH(item_json) across the whole archive
+      // is a full scan that only grows. Average the newest 1,000 rows (written
+      // under the current content cap) and multiply by the row count.
+      const [countRow, avgRow] = await Promise.all([
+        db.prepare('SELECT COUNT(*) as count FROM feed_items').first<{ count: number }>(),
+        db
+          .prepare(
+            `SELECT AVG(LENGTH(item_json)) as avg
+               FROM (SELECT item_json FROM feed_items ORDER BY seq DESC LIMIT 1000)`
+          )
+          .first<{ avg: number | null }>(),
+      ]);
+      const bytes = Math.round((countRow?.count ?? 0) * (avgRow?.avg ?? 0));
+      return {
+        label: 'Archive Size (est.)',
+        value: (bytes / (1024 * 1024 * 1024)).toFixed(2),
+        unit: 'GB',
+        status: bytes > ARCHIVE_ALERT_BYTES ? 'warning' : 'healthy',
+      };
     },
   },
   {
-    id: 'feeds_with_errors',
+    id: 'churn_feeds',
     category: 'Feeds',
     query: async (db) => {
       const r = await db
-        .prepare('SELECT COUNT(*) as count FROM feed_metadata WHERE error_count > 0')
+        .prepare(
+          `SELECT COUNT(*) as count FROM (
+             SELECT feed_url FROM feed_items GROUP BY feed_url HAVING COUNT(*) > ?
+           )`
+        )
+        .bind(CHURN_WARN_ITEMS)
         .first<{ count: number }>();
       const count = r?.count ?? 0;
       return {
-        label: 'Feeds with Errors',
+        label: 'Feeds Near Sanity Cap',
         value: count,
         status: count > 0 ? 'warning' : 'healthy',
       };
@@ -31,7 +147,12 @@ export const feedMetrics: MetricDefinition[] = [
     category: 'Feeds',
     query: async (db) => {
       const r = await db
-        .prepare('SELECT ROUND(AVG(subscriber_count), 1) as avg FROM feed_metadata')
+        .prepare(
+          `SELECT ROUND(AVG(subs), 1) as avg FROM (
+             SELECT COUNT(*) as subs FROM subscriptions_cache
+              WHERE active = 1 GROUP BY feed_url
+           )`
+        )
         .first<{ avg: number }>();
       return { label: 'Avg Subscribers/Feed', value: r?.avg ?? 0 };
     },

admin/src/lib/metrics/ops.ts

@@ -161,7 +161,10 @@ const seriesDefinitions: { key: keyof SnapshotRow; label: string; unit?: string
   { key: 'users', label: 'Users' },
   { key: 'active_sessions', label: 'Active Sessions' },
   { key: 'feeds', label: 'Feeds' },
-  { key: 'feeds_with_errors', label: 'Feeds with Errors' },
+  // Same number as the live tile above, hour by hour: per-feed fetch errors are
+  // the crawler's to know, so this series is empty for any hour whose proxy
+  // stats were missing or stale.
+  { key: 'feeds_with_errors', label: 'Proxy Feeds in Error' },
   { key: 'subscriptions', label: 'Subscriptions' },
   { key: 'saved_articles', label: 'Saved Articles' },
   { key: 'feed_items', label: 'Feed Items' },

admin/src/lib/metrics/system.ts

@@ -2,7 +2,7 @@ import type { MetricDefinition } from '$lib/types';
 
 const rowCountTables = [
   { table: 'users', label: 'Users' },
-  { table: 'feed_metadata', label: 'Feeds' },
+  { table: 'feeds', label: 'Feeds' },
   { table: 'feed_items', label: 'Feed Items' },
   { table: 'subscriptions_cache', label: 'Subscriptions' },
 ];