From b8f48e0649300e1ec328f051a82f3154e5fc9066 Mon Sep 17 00:00:00 2001 From: Maruthi Prithivi Date: Wed, 29 Jul 2026 17:56:00 +0800 Subject: [PATCH 1/2] feat(workshop): add Polymarket analytics track --- .github/workflows/workshop.yml | 43 +- workshops/build_workshop/README.md | 39 +- workshops/build_workshop/playbook/Dockerfile | 4 + .../playbook/content/docs/ai-sre.mdx | 29 + .../playbook/content/docs/index.mdx | 245 +----- .../content/docs/instructor/meta.json | 2 +- .../playbook/content/docs/learner/meta.json | 2 +- .../playbook/content/docs/meta.json | 2 +- .../content/docs/polymarket/index.mdx | 59 ++ .../docs/polymarket/instructor/00-setup.mdx | 34 + .../instructor/01-discover-markets.mdx | 26 + .../polymarket/instructor/02-model-data.mdx | 33 + .../instructor/03-stream-live-data.mdx | 32 + .../instructor/04-realtime-aggregates.mdx | 28 + .../instructor/05-investigate-movement.mdx | 25 + .../instructor/06-cloud-dashboard.mdx | 29 + .../docs/polymarket/instructor/07-wrap-up.mdx | 30 + .../docs/polymarket/instructor/index.mdx | 39 + .../docs/polymarket/instructor/meta.json | 16 + .../docs/polymarket/learner/00-setup.mdx | 182 ++++ .../learner/01-discover-markets.mdx | 78 ++ .../docs/polymarket/learner/02-model-data.mdx | 162 ++++ .../learner/03-stream-live-data.mdx | 109 +++ .../learner/04-realtime-aggregates.mdx | 82 ++ .../learner/05-investigate-movement.mdx | 117 +++ .../polymarket/learner/06-cloud-dashboard.mdx | 93 ++ .../docs/polymarket/learner/07-wrap-up.mdx | 57 ++ .../content/docs/polymarket/learner/index.mdx | 27 + .../content/docs/polymarket/learner/meta.json | 17 + .../polymarket/learner/troubleshooting.mdx | 81 ++ .../content/docs/polymarket/meta.json | 6 + .../content/docs/polymarket/rehearsal.mdx | 25 + .../src/app/docs/[[...slug]]/page.tsx | 8 +- .../app/llms.mdx/docs/[[...slug]]/route.ts | 8 +- .../src/app/og/docs/[...slug]/route.tsx | 8 +- .../playbook/src/components/mdx.tsx | 5 + .../playbook/src/lib/layout.shared.tsx | 11 +- .../build_workshop/playbook/src/lib/shared.ts | 2 +- .../build_workshop/playbook/src/lib/source.ts | 7 + .../polymarket/.env.polymarket.example | 18 + .../build_workshop/polymarket/.gitignore | 3 + workshops/build_workshop/polymarket/README.md | 60 ++ .../polymarket/collector/Dockerfile | 18 + .../collector/collector/__init__.py | 1 + .../collector/collector/__main__.py | 40 + .../polymarket/collector/collector/api.py | 185 ++++ .../polymarket/collector/collector/config.py | 71 ++ .../polymarket/collector/collector/health.py | 58 ++ .../polymarket/collector/collector/models.py | 398 +++++++++ .../polymarket/collector/collector/service.py | 797 ++++++++++++++++++ .../polymarket/collector/collector/storage.py | 173 ++++ .../polymarket/collector/requirements-dev.txt | 3 + .../polymarket/collector/requirements.txt | 2 + .../polymarket/collector/tests/test_api.py | 226 +++++ .../polymarket/collector/tests/test_config.py | 40 + .../polymarket/collector/tests/test_models.py | 267 ++++++ .../collector/tests/test_service.py | 433 ++++++++++ .../tests/test_storage_integration.py | 131 +++ .../build_workshop/polymarket/db/queries.sql | 92 ++ .../build_workshop/polymarket/db/schema.sql | 95 +++ .../polymarket/docker-compose.yml | 38 + .../build_workshop/polymarket/preflight.sh | 83 ++ .../polymarket/test-clickhouse.sh | 136 +++ .../build_workshop/scripts/check-docs.sh | 54 +- .../scripts/check-polymarket-sql.py | 49 ++ .../build_workshop/scripts/check-windows.ps1 | 7 + 66 files changed, 5040 insertions(+), 240 deletions(-) create mode 100644 workshops/build_workshop/playbook/content/docs/ai-sre.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/index.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/00-setup.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/01-discover-markets.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/02-model-data.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/03-stream-live-data.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/04-realtime-aggregates.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/05-investigate-movement.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/06-cloud-dashboard.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/07-wrap-up.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/index.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/instructor/meta.json create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/00-setup.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/01-discover-markets.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/02-model-data.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/03-stream-live-data.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/04-realtime-aggregates.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/05-investigate-movement.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/06-cloud-dashboard.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/07-wrap-up.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/index.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/meta.json create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/learner/troubleshooting.mdx create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/meta.json create mode 100644 workshops/build_workshop/playbook/content/docs/polymarket/rehearsal.mdx create mode 100644 workshops/build_workshop/polymarket/.env.polymarket.example create mode 100644 workshops/build_workshop/polymarket/.gitignore create mode 100644 workshops/build_workshop/polymarket/README.md create mode 100644 workshops/build_workshop/polymarket/collector/Dockerfile create mode 100644 workshops/build_workshop/polymarket/collector/collector/__init__.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/__main__.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/api.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/config.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/health.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/models.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/service.py create mode 100644 workshops/build_workshop/polymarket/collector/collector/storage.py create mode 100644 workshops/build_workshop/polymarket/collector/requirements-dev.txt create mode 100644 workshops/build_workshop/polymarket/collector/requirements.txt create mode 100644 workshops/build_workshop/polymarket/collector/tests/test_api.py create mode 100644 workshops/build_workshop/polymarket/collector/tests/test_config.py create mode 100644 workshops/build_workshop/polymarket/collector/tests/test_models.py create mode 100644 workshops/build_workshop/polymarket/collector/tests/test_service.py create mode 100644 workshops/build_workshop/polymarket/collector/tests/test_storage_integration.py create mode 100644 workshops/build_workshop/polymarket/db/queries.sql create mode 100644 workshops/build_workshop/polymarket/db/schema.sql create mode 100644 workshops/build_workshop/polymarket/docker-compose.yml create mode 100755 workshops/build_workshop/polymarket/preflight.sh create mode 100755 workshops/build_workshop/polymarket/test-clickhouse.sh create mode 100644 workshops/build_workshop/scripts/check-polymarket-sql.py diff --git a/.github/workflows/workshop.yml b/.github/workflows/workshop.yml index d11900f..41d45e1 100644 --- a/.github/workflows/workshop.yml +++ b/.github/workflows/workshop.yml @@ -43,6 +43,8 @@ jobs: cache-dependency-path: | workshops/build_workshop/app/backend/requirements.txt workshops/build_workshop/app/backend/requirements-dev.txt + workshops/build_workshop/polymarket/collector/requirements.txt + workshops/build_workshop/polymarket/collector/requirements-dev.txt - name: Run backend tests working-directory: workshops/build_workshop/app/backend @@ -50,6 +52,16 @@ jobs: python -m pip install -r requirements-dev.txt python -m pytest tests/test_chat_guardrails.py tests/test_db_retry.py tests/test_cloud_defaults.py ../loadgen/test_config.py -q + - name: Run Polymarket collector tests + working-directory: workshops/build_workshop/polymarket/collector + run: | + python -m pip install -r requirements-dev.txt + python -m pytest -q + + - name: Run Polymarket ClickHouse integration + working-directory: workshops/build_workshop + run: ./polymarket/test-clickhouse.sh + - name: Run RTA dashboard tests working-directory: workshops/RTA-mini-workshop/dashboard run: python -m pytest test_app.py -q @@ -73,6 +85,8 @@ jobs: env: NEXT_PUBLIC_BASE_PATH: "" NEXT_PUBLIC_SITE_URL: https://workshop.demohouse.cloud + NEXT_PUBLIC_WORKSHOP_ENV: dev + NEXT_PUBLIC_WORKSHOP_BRANCH: dev-build-workshop-v1 NEXT_TELEMETRY_DISABLED: "1" run: npm run build -- --webpack @@ -102,6 +116,8 @@ jobs: cache-dependency-path: | workshops/build_workshop/app/backend/requirements.txt workshops/build_workshop/app/backend/requirements-dev.txt + workshops/build_workshop/polymarket/collector/requirements.txt + workshops/build_workshop/polymarket/collector/requirements-dev.txt - name: Run backend tests on Windows working-directory: workshops/build_workshop/app/backend @@ -110,6 +126,13 @@ jobs: python -m pip install -r requirements-dev.txt python -m pytest tests/test_chat_guardrails.py tests/test_db_retry.py tests/test_cloud_defaults.py ../loadgen/test_config.py -q + - name: Run Polymarket collector tests on Windows + working-directory: workshops/build_workshop/polymarket/collector + shell: pwsh + run: | + python -m pip install -r requirements-dev.txt + python -m pytest -q + - name: Run RTA dashboard tests on Windows working-directory: workshops/RTA-mini-workshop/dashboard shell: pwsh @@ -163,10 +186,12 @@ jobs: echo "instance_id=${PROD_INSTANCE_ID}" >> "${GITHUB_OUTPUT}" echo "image_alias=prod" >> "${GITHUB_OUTPUT}" echo "url=https://workshop.demohouse.cloud" >> "${GITHUB_OUTPUT}" + echo "workshop_env=prod" >> "${GITHUB_OUTPUT}" else echo "instance_id=${DEV_INSTANCE_ID}" >> "${GITHUB_OUTPUT}" echo "image_alias=dev" >> "${GITHUB_OUTPUT}" echo "url=https://dev-workshop.demohouse.cloud" >> "${GITHUB_OUTPUT}" + echo "workshop_env=dev" >> "${GITHUB_OUTPUT}" fi test -n "$(sed -n 's/^instance_id=//p' "${GITHUB_OUTPUT}")" @@ -192,11 +217,15 @@ jobs: REPOSITORY: posthouse-demo-workshop IMAGE_ALIAS: ${{ steps.target.outputs.image_alias }} SITE_URL: ${{ steps.target.outputs.url }} + WORKSHOP_ENV: ${{ steps.target.outputs.workshop_env }} + WORKSHOP_BRANCH: ${{ github.ref_name }} run: | docker buildx build \ --platform linux/arm64 \ --build-arg NEXT_PUBLIC_BASE_PATH= \ --build-arg NEXT_PUBLIC_SITE_URL="${SITE_URL}" \ + --build-arg NEXT_PUBLIC_WORKSHOP_ENV="${WORKSHOP_ENV}" \ + --build-arg NEXT_PUBLIC_WORKSHOP_BRANCH="${WORKSHOP_BRANCH}" \ --tag "${REGISTRY}/${REPOSITORY}:${GITHUB_SHA}" \ --tag "${REGISTRY}/${REPOSITORY}:${IMAGE_ALIAS}" \ --push \ @@ -272,6 +301,7 @@ jobs: - name: Verify public routes env: SITE_URL: ${{ steps.target.outputs.url }} + WORKSHOP_ENV: ${{ steps.target.outputs.workshop_env }} run: | set -euo pipefail @@ -306,10 +336,21 @@ jobs: return 1 } - for path in / /build-workshop /rta-mini/index.html /docs/learner/00-setup /docs/learner/07-break-and-fix /docs/learner/08-chat-langfuse; do + routes=(/ /build-workshop /rta-mini/index.html /docs/ai-sre /docs/polymarket /docs/polymarket/learner/00-setup /docs/polymarket/instructor /docs/learner/00-setup /docs/learner/07-break-and-fix /docs/learner/08-chat-langfuse) + if [[ "${WORKSHOP_ENV}" == "dev" ]]; then + routes+=(/docs/polymarket/rehearsal) + fi + for path in "${routes[@]}"; do probe_route "${path}" done + if [[ "${WORKSHOP_ENV}" == "prod" ]]; then + rehearsal_status=$(curl --show-error --silent \ + --output /dev/null --write-out '%{http_code}' \ + "${SITE_URL}/docs/polymarket/rehearsal") + test "${rehearsal_status}" = "404" + fi + - name: Deployment summary env: SITE_URL: ${{ steps.target.outputs.url }} diff --git a/workshops/build_workshop/README.md b/workshops/build_workshop/README.md index 7235a29..a44743e 100644 --- a/workshops/build_workshop/README.md +++ b/workshops/build_workshop/README.md @@ -1,10 +1,15 @@ -# ClickHouse BUILD Workshop ("Build AI with AI") +# ClickHouse Cloud use-case workshops -A three-hour, hands-on workshop: participants use their own agentic coding tool to take -an NYC-taxi ride-hailing analytics app end to end on ClickHouse Cloud — CDC ingestion -with ClickPipes, conversational BI with ClickHouse Agents, observability with ClickStack -(including an AI-built SRE dashboard), a break-and-fix incident lab diagnosed by an AI -SRE, and an in-app AI chat traced to Langfuse Cloud. +This site now carries two dedicated use cases: + +- **AI SRE with NYC taxi data:** the existing three-hour application, managed Postgres + CDC, Agents, ClickStack, incident diagnosis, and Langfuse journey under `app/`. +- **Polymarket real-time analytics:** a two-hour public market-data stream, typed + ClickHouse model, one-minute aggregate, investigation, and Cloud dashboard under + `polymarket/`. + +Both tracks support macOS and Windows through Ubuntu on WSL 2. ClickHouse Cloud is the +only ClickHouse server; no local database substitute is part of either track. Learners clone the repository, then switch to `build-workshop-v1`. Maintainers create a feature branch, open a PR to protected `dev-build-workshop-v1`, verify @@ -12,7 +17,7 @@ feature branch, open a PR to protected `dev-build-workshop-v1`, verify `dev-build-workshop-v1` to protected `build-workshop-v1` for [workshop.demohouse.cloud](https://workshop.demohouse.cloud). -## Architecture +## AI SRE architecture The app edge runs on the participant's laptop. ClickHouse, Postgres, ClickPipes, and ClickStack/HyperDX live in ClickHouse Cloud; OpenAI and Langfuse are separate hosted @@ -121,7 +126,8 @@ flowchart LR | Path | What | |---|---| | `app/` | The local application edge: React frontend, FastAPI backend, managed-Postgres data generator, and stateless ClickStack OTel forwarder. All databases and product UIs are cloud-hosted. Workshop entrypoint: `preflight.sh` + `docker-compose.workshop.yml` + `.env.workshop.example`. | -| `playbook/` | The published follow-along playbook (Next.js + Fumadocs; dual learner/instructor tracks plus self-paced and troubleshooting pages; deploys to workshop.demohouse.cloud). Requires Node >= 22.12 to build. | +| `polymarket/` | Public Polymarket collector, ClickHouse schema/reference queries, fixture mode, Docker Compose entrypoint, and tests. The only Compose service is the stateless collector. | +| `playbook/` | The published workshop catalog and dedicated AI SRE / Polymarket learner and instructor tracks. Requires Node >= 22.12 to build. | | `docs/` | `diagrams/` — the platform, architecture, data-flow, and module-flow SVGs, generated by `gen_diagrams.py`. | | `infra/` | Instructor tooling via clickhousectl: the demo stack end-to-end run and a cloud-hosted managed-Postgres fallback pool for participants whose orgs cannot create one. | @@ -138,7 +144,7 @@ The learner playbook lists the branch names and the observable symptom of each f diagnosis paths and fixes live in the playbook's instructor track (module 07) and are deliberately not documented in this directory. -## Quick start (participant) +## AI SRE quick start ```bash git clone @@ -152,3 +158,18 @@ docker compose --env-file .env.workshop -f docker-compose.workshop.yml up -d Then follow the playbook from module 00 (or the self-paced page if no instructor is around). + +## Polymarket quick start + +```bash +git clone +cd ClickHouse_Demos +git switch build-workshop-v1 +cd workshops/build_workshop/polymarket +cp .env.polymarket.example .env.polymarket +set -a; source ./.env.polymarket; set +a +./preflight.sh +``` + +Then start at `/docs/polymarket/learner/00-setup`. Learners create the schema from +copyable SQL on the site before starting the collector. diff --git a/workshops/build_workshop/playbook/Dockerfile b/workshops/build_workshop/playbook/Dockerfile index 00b3edf..403742b 100644 --- a/workshops/build_workshop/playbook/Dockerfile +++ b/workshops/build_workshop/playbook/Dockerfile @@ -25,8 +25,12 @@ FROM node:22-slim AS builder WORKDIR /app ARG NEXT_PUBLIC_BASE_PATH= ARG NEXT_PUBLIC_SITE_URL=https://dev-workshop.demohouse.cloud +ARG NEXT_PUBLIC_WORKSHOP_ENV=prod +ARG NEXT_PUBLIC_WORKSHOP_BRANCH=build-workshop-v1 ENV NEXT_PUBLIC_BASE_PATH=$NEXT_PUBLIC_BASE_PATH ENV NEXT_PUBLIC_SITE_URL=$NEXT_PUBLIC_SITE_URL +ENV NEXT_PUBLIC_WORKSHOP_ENV=$NEXT_PUBLIC_WORKSHOP_ENV +ENV NEXT_PUBLIC_WORKSHOP_BRANCH=$NEXT_PUBLIC_WORKSHOP_BRANCH ENV NEXT_TELEMETRY_DISABLED=1 COPY --from=deps /app/node_modules ./node_modules COPY . . diff --git a/workshops/build_workshop/playbook/content/docs/ai-sre.mdx b/workshops/build_workshop/playbook/content/docs/ai-sre.mdx new file mode 100644 index 0000000..bf5422f --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/ai-sre.mdx @@ -0,0 +1,29 @@ +--- +title: AI SRE track +description: The existing three-hour NYC taxi application, observability, incident, and AI workflow. +--- + +Build a real-time NYC taxi application on ClickHouse Cloud, stream changes from managed +Postgres with ClickPipes, observe the app with ClickStack, diagnose a fault with an AI +SRE workflow, and trace the application chat with Langfuse. + +## Choose your role + + + + + + +## Outcome + +In about three hours you build and verify: + +- a ClickHouse Cloud analytics service with millions of taxi rows; +- managed Postgres CDC through ClickPipes; +- ClickHouse Agents over the operational data; +- application traces and logs in Managed ClickStack; +- an AI-assisted SRE dashboard, alert, and incident diagnosis; and +- an in-app AI chat traced in Langfuse Cloud. + +The existing learner and instructor URLs remain unchanged so saved links continue to +work. diff --git a/workshops/build_workshop/playbook/content/docs/index.mdx b/workshops/build_workshop/playbook/content/docs/index.mdx index a7cbdbb..38f8fa0 100644 --- a/workshops/build_workshop/playbook/content/docs/index.mdx +++ b/workshops/build_workshop/playbook/content/docs/index.mdx @@ -1,227 +1,52 @@ --- -title: Build AI with AI -description: A three-hour, hands-on workshop that takes an NYC-taxi analytics app end to end on ClickHouse Cloud, driven by your own AI coding agent. +title: ClickHouse Cloud workshops +description: Choose a use case, then follow its learner or instructor track. --- -Welcome to the ClickHouse BUILD Workshop playbook. Over three hours you will take an -NYC-taxi ride-hailing analytics app (React front end, FastAPI back end, Postgres source -database) and stand it up end to end on ClickHouse Cloud, using your own agentic coding -tool to do the building. By the end you will have real-time change data capture from a -managed Postgres, conversational BI over your data, full observability, an AI-assisted -SRE workflow, and an in-app AI chat traced end to end, and you will have practiced -diagnosing a live incident with an AI SRE. +One workshop site, two complete use cases. Both run the stateful data plane in +ClickHouse Cloud and support macOS or Windows with Ubuntu on WSL 2. -The same workshop supports **macOS** and **Windows**. Choose your computer in the page -header once; the playbook keeps the correct setup visible everywhere. Windows runs the -shared workshop toolchain inside Ubuntu on WSL 2. - -You leave with a running prototype and this repo to show your team. Managed ClickHouse, -Postgres, ClickPipes, Agents, and ClickStack artifacts persist for the trial; Langfuse -traces persist in Langfuse Cloud. Restart the local app and telemetry forwarder with -Docker when you return. By the end you can demo five things live: - -- a real-time operational dashboard on ClickHouse Cloud, -- a change-data-capture pipeline streaming from managed Postgres, -- conversational BI over your data with ClickHouse Agents, -- an AI-built SRE dashboard and alert over your telemetry, -- and an in-app AI chat traced end to end in Langfuse. - -This playbook is dual-track. The Learner track is the lesson you follow in the room. The -Instructor track is the facilitator's companion for the same module: timing, talk -track, common failures, and reset steps. +## Choose a use case - - + + +## What both tracks guarantee -## For learners - -You will work module by module. Each module names a starting checkpoint, explains why -the step matters, states a concrete goal, walks you through numbered steps, and ends -with a verification you can check yourself. You never have to keep pace with the whole -room: if you fall behind, each module's Starting point section states exactly what you -need in place, so you can catch up from there at your own speed. - -What you need on the day: - -- A laptop that meets the prerequisites in [00 Setup](/docs/learner/00-setup). -- Your own agentic coding tool (Claude Code, Cursor, Codex CLI, or Windsurf), signed in - and on an active plan. -- A ClickHouse Cloud account with visible trial credits (created during prework). -- The workshop app repository cloned locally, with Docker running. - -**Self-paced?** You can complete the whole workshop with no instructor - the primary path -is fully self-serve. See [Running this workshop self-paced](/docs/learner/self-paced) for -what changes, how to use your coding agent as an instructor, and the -[Troubleshooting](/docs/learner/troubleshooting) reference for every failure seen in -testing. - -## For instructors - -The Instructor track mirrors the Learner track one-to-one. For every module you get a -timing budget, a talk track, the failures that actually happen in the room and how to -unstick them, and the exact reset steps to get an attendee (or the whole room) back to a -known-good state. Read the [Instructor track landing page](/docs/instructor) first for -the room-level run of show and the shared-resource checklist. - -## Workshop scope - -### The app you build on - -The workshop app is a self-contained "war room" analytics stack for an NYC-taxi -ride-hailing business: - -- **Front end** — a React single-page app with two dashboards: an Ops dashboard - (live operational metrics) and a Historical dashboard (large-range aggregations and - drilldowns). It also hosts the in-app AI chat panel you wire up in module 08. -- **Back end** — a FastAPI service exposing safe, parameterized analytics endpoints. -- **Source database** — ClickHouse-managed Postgres as the operational source of truth, - the origin for change data capture into ClickHouse. -- **Analytics warehouse** — ClickHouse Cloud, which you point the back end at in - module 01 and which receives CDC, observability data, and BI queries. - -You do not build the app from scratch. The React/FastAPI app runs locally, while every -stateful data or product service is cloud-hosted from the start. - -### The platform you build on - -ClickHouse Cloud is a single platform that spans the whole stack — from ingesting data at -the bottom, to storing and analyzing it in ClickHouse, to observing it and layering AI on -top. This workshop is a guided tour through exactly these pieces: you ingest with -ClickPipes, analyze in ClickHouse, and observe with Managed ClickStack/HyperDX. Langfuse -and OpenAI are separate hosted services. - -![The ClickHouse Cloud platform: layered stack from sources and ClickPipes ingestion, through ClickHouse and Postgres, up to HyperDX, Langfuse and agentic AI](/clickhouse-platform.svg) - -### Architecture (target end state) - -First, group the pieces by where they run: the stateless app and tools on your laptop, -stateful services in ClickHouse Cloud, and the separate hosted AI services. - -![Workshop components grouped into participant laptop, ClickHouse Cloud, and external hosted services](/workshop-architecture.svg) - -### How data flows - -Now follow one live trip from the local load generator through managed Postgres and -ClickPipes, into `default.realtime_trips`, and finally into the application dashboards. - -![A live trip flows from the load generator to managed Postgres, through ClickPipes into default.realtime_trips, then through a materialized view to the app](/workshop-data-flow.svg) - -The diagrams above are generated from `workshops/build_workshop/docs/diagrams/gen_diagrams.py` -(edit the script and re-run it to regenerate the SVGs) — see the module flow below. - -![Module flow: ten core workshop modules](/workshop-module-flow.svg) - -## Modules - -Ten core modules, in order. The app is complete on `build-workshop-v1`, so every module except -07 needs no per-module checkout — you configure and connect services rather than change -app code. The only branch switches are the fault branches in module 07. - -| Step | Time | Learner lesson | Instructor notes | Branch | What you will learn | -|---|---|---|---|---|---| -| 00 | 25 min | [Setup](/docs/learner/00-setup) | [notes](/docs/instructor/00-setup) | `build-workshop-v1` | Accounts, tools, agent skills, and the app repo, all wired up and verified | -| 01 | 15 min | [ClickHouse Cloud](/docs/learner/01-clickhouse-cloud) | [notes](/docs/instructor/01-clickhouse-cloud) | `build-workshop-v1` | Create the schema, seed historical data from object storage, and feel the query speed | -| 02 | 5 min | [Base app](/docs/learner/02-base-app) | [notes](/docs/instructor/02-base-app) | `build-workshop-v1` | Tour the running app now that it has data: Ops and Historical dashboards, the chat panel, the data flow | -| 03 | 20 min | [Managed Postgres CDC](/docs/learner/03-realtime-cdc) | [notes](/docs/instructor/03-realtime-cdc) | `build-workshop-v1` | Stream live rows from ClickHouse-managed Postgres into ClickHouse with a Postgres CDC ClickPipe | -| 04 | 10 min | [ClickHouse Agents](/docs/learner/04-clickhouse-agents) | [notes](/docs/instructor/04-clickhouse-agents) | `build-workshop-v1` | Conversational BI: create an agent over your taxi data and explore it in natural language | -| 05 | 15 min | [ClickStack](/docs/learner/05-clickstack) | [notes](/docs/instructor/05-clickstack) | `build-workshop-v1` | Enable ClickStack and send app traces and logs to HyperDX | -| 06 | 15 min | [AI SRE](/docs/learner/06-ai-sre) | [notes](/docs/instructor/06-ai-sre) | `build-workshop-v1` | Use the ClickStack MCP connection to build an SRE dashboard and alert | -| 07 | 20 min | [Test, fail, and fix](/docs/learner/07-break-and-fix) | [notes](/docs/instructor/07-break-and-fix) | `fault/*` | Continue from AI SRE, inject a fault, diagnose it, fix it, and prove recovery | -| 08 | 15 min | [Chat and Langfuse](/docs/learner/08-chat-langfuse) | [notes](/docs/instructor/08-chat-langfuse) | `build-workshop-v1` | Use the in-app AI chat and follow its traces, generations, and costs in Langfuse | -| 09 | 10 min | [Wrap-up](/docs/learner/09-wrap-up) | [notes](/docs/instructor/09-wrap-up) | `build-workshop-v1` | Review what you built, take it home, and extend it to your own data | - -The times add up to about 2 hours 30 minutes of hands-on work; the rest of the -three-hour session is the opening, transitions, and the finale demos. - -## How to work through it - -### Branches - -The app is already complete on `build-workshop-v1`. This workshop is about configuring -and connecting services — real-time CDC, observability, agents, chat — not editing -application code, so there is nothing to build up branch by branch: - -- Clone the app once in module 00, switch to `build-workshop-v1`, and stay there except - while running the fault scenarios in module 07. -- Late joiners are never stranded: managed resources are service-side, and the complete - local app restarts from the workshop branch plus `.env.workshop`. There is no - per-module checkout to catch up on. -- The only branch switches are in module 07 (Break and fix), which uses fault branches - (`fault/01-map-not-loading`, `fault/02-zone-stats-500`, `fault/03-slow-dashboard`). You - check one out, diagnose the failure, then preserve the fix and return to - `build-workshop-v1` with the module 07 stash-and-switch reset. - - - The three `fault/*` branches above exist in the repo; module 07 walks through checking - one out. Their symptoms and fixes are documented in the instructor track's answer key. - - -### Environment variables - -Workshop configuration lives in a single `.env.workshop` file at the root of the app -repository. A safe `.env.workshop.example` is committed; copy it and fill in only the -values that are yours: - -```bash -cp .env.workshop.example .env.workshop -``` - -The stack reads it explicitly via `--env-file`: - -```bash -docker compose --env-file .env.workshop -f docker-compose.workshop.yml up -d -``` - -You fill in the values during module 00 setup, except the ClickStack block, which you add -in module 05: - -- `CLICKHOUSE_HOST`, `CLICKHOUSE_PASSWORD` (plus `CLICKHOUSE_PORT=8443`, - `CLICKHOUSE_USER=default`, `CLICKHOUSE_DATABASE=nyc_tlc_data`, `CLICKHOUSE_SECURE=true`) - — your Cloud service (module 00). `CLICKHOUSE_HOST` is the bare hostname, with no - scheme and no port. -- `OPENAI_API_KEY`, `LLM_MODEL=gpt-5.4-mini`, `LLM_BASE_URL` — the runtime LLM for the - in-app chat (module 00). -- `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_BASE_URL` — your Langfuse - project (module 00). `LANGFUSE_BASE_URL` is the Langfuse v4 env name; use - `https://us.cloud.langfuse.com` (US) or `https://cloud.langfuse.com` (EU). -- `OTLP_AUTH_TOKEN`, `CLICKSTACK_DATABASE=otel`, `OTEL_SERVICE_NAME=nyc-taxi-backend` - — observability, in the ClickStack section of the same `.env.workshop` (module 05). +- Every stateful data service is cloud-hosted. Your laptop runs only clients, the + application or collector, and other stateless workshop processes. +- Every command and SQL statement appears as a copyable block on this site. +- Every module ends with an objective check and a known handoff to the next module. +- Learner and instructor pages follow the same module sequence. +- The public-data exercises do not place trades or require a Polymarket wallet. -Never commit your filled-in `.env.workshop`; it is git-ignored. +## Pick your role -## Repository layout +Each use case has two audiences: -Everything lives in one repository (`ClickHouse_Demos`), under -`workshops/build_workshop/`, on the `build-workshop-v1` branch. +- **Learner:** the exact hands-on path, with commands, SQL, expected output, and recovery. +- **Instructor:** timing, talk track, classroom failure modes, fixture fallback, and reset steps. -```text -workshops/build_workshop/ -playbook/ # this playbook (the site you are reading) - content/docs/ - index.mdx # this overview - learner/ # self-paced guide, the lessons 00-setup ... 09-wrap-up, - # and a troubleshooting reference - instructor/ # facilitator notes: 00-setup ... 09-wrap-up - src/ # Next.js + Fumadocs app - README.md # run, build, deploy, and authoring guide +Start by choosing the use case above. Do not mix modules from different use cases in one +run; they create different databases and have different completion checks. -app/ # the NYC-taxi app you build on (cloned in module 00) - frontend/ # React SPA (Ops + Historical dashboards, chat panel) - backend/ # FastAPI analytics API + AI chat - db/cloud/001_cloud_schema.sql # maintainer fixture; Module 01 contains the copyable SQL - docker-compose.workshop.yml # the workshop stack (Cloud + ClickPipes) - docker-compose.otel.yml # the ClickStack observability overlay (module 05) - .env.workshop.example # single env template (Cloud + chat + observability); - # copy to .env.workshop and fill in your values -``` +## Platform -## Where to go next +ClickHouse Cloud provides the service, SQL console, saved queries, and dashboards used +by both tracks. The AI SRE track also uses managed Postgres, ClickPipes, ClickStack, +ClickHouse Agents, and Langfuse Cloud. The Polymarket track uses public market-data APIs +and writes directly to ClickHouse Cloud with acknowledged asynchronous inserts. - - - - + + diff --git a/workshops/build_workshop/playbook/content/docs/instructor/meta.json b/workshops/build_workshop/playbook/content/docs/instructor/meta.json index 8c9386c..117582d 100644 --- a/workshops/build_workshop/playbook/content/docs/instructor/meta.json +++ b/workshops/build_workshop/playbook/content/docs/instructor/meta.json @@ -1,5 +1,5 @@ { - "title": "Instructor track", + "title": "AI SRE · Instructor", "description": "Timing, talk tracks, common failures, and reset steps for facilitators.", "root": true, "icon": "Presentation", diff --git a/workshops/build_workshop/playbook/content/docs/learner/meta.json b/workshops/build_workshop/playbook/content/docs/learner/meta.json index 858d7a9..2c71147 100644 --- a/workshops/build_workshop/playbook/content/docs/learner/meta.json +++ b/workshops/build_workshop/playbook/content/docs/learner/meta.json @@ -1,5 +1,5 @@ { - "title": "Learner track", + "title": "AI SRE · Learner", "description": "The hands-on lessons you work through during the session.", "root": true, "icon": "GraduationCap", diff --git a/workshops/build_workshop/playbook/content/docs/meta.json b/workshops/build_workshop/playbook/content/docs/meta.json index 45c0626..5106320 100644 --- a/workshops/build_workshop/playbook/content/docs/meta.json +++ b/workshops/build_workshop/playbook/content/docs/meta.json @@ -1,3 +1,3 @@ { - "pages": ["index", "learner", "instructor"] + "pages": ["index", "ai-sre", "polymarket", "learner", "instructor"] } diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/index.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/index.mdx new file mode 100644 index 0000000..9a74212 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/index.mdx @@ -0,0 +1,59 @@ +--- +title: Polymarket real-time analytics +description: Stream public prediction-market data, model it in ClickHouse Cloud, and publish a market pulse dashboard. +--- + +Watch a public market move, see the update land in your own ClickHouse Cloud service, +and explain whether the move came with a wider spread or faster trading. + +This is a **market-data workshop**, not a trading tutorial. It uses only public Gamma, +CLOB, and Data APIs. You do not create a wallet, deposit funds, place an order, or provide +a Polymarket secret. + +## Choose your role + + + + + + +## What you build + +```text +Gamma market discovery ─┐ + ├─ stateless collector ─ ClickHouse Cloud ─ saved queries +CLOB WebSocket + REST ──┤ │ +Data API trades ────────┘ └─ 1-minute midpoint MV ─ dashboard +``` + +- `polymarket.markets`: the watched markets and their Yes/No token metadata. +- `polymarket.price_ticks`: live quote, price-change, and top-of-book observations. +- `polymarket.trades`: reconciled public trades with deterministic IDs. +- `polymarket.market_midpoints_1m`: quote-midpoint OHLC maintained on insert. +- A Cloud dashboard for current probability, movers, spread/freshness, volume velocity, + and one-minute midpoint movement. + +## Reliability is part of the lesson + +The WebSocket is fast but is not treated as a durable log. The collector detects a +quiet or stalled socket, reconnects, and polls the public CLOB book while degraded. A +separate Data API loop reconciles trades with a five-second overlap and deterministic +deduplication. If the classroom network blocks Polymarket, fixture mode produces the +same schema and dashboard flow without pretending the feed is live. + +## Time + +About **2 hours 10 minutes** hands-on: + +| Module | Outcome | Time | +|---|---|---:| +| 00 | Prepare the laptop, Cloud service, and environment | 15 min | +| 01 | Discover active markets and token IDs | 10 min | +| 02 | Create query-driven ClickHouse tables and MV | 20 min | +| 03 | Start and verify the resilient live collector | 15 min | +| 04 | Understand and query the one-minute aggregate | 20 min | +| 05 | Investigate probability, spread, freshness, and volume | 20 min | +| 06 | Publish the Cloud market pulse dashboard | 20 min | +| 07 | Prove completion, clean up, and map the production path | 10 min | + +Start with [Module 00](/docs/polymarket/learner/00-setup). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/00-setup.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/00-setup.mdx new file mode 100644 index 0000000..89164d1 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/00-setup.mdx @@ -0,0 +1,34 @@ +--- +title: 00 Setup · Instructor +description: Timing, talk track, failures, and reset for Polymarket setup. +--- + +## Timing + +15 minutes. Start service creation first; explain the architecture while Cloud becomes ready. + +## Talk track + +- Windows means Ubuntu on WSL 2. PowerShell is only for installing or inspecting WSL. +- `clickhousectl local use stable` installs the client binary, not a local server. +- `.env.polymarket` contains only ClickHouse Cloud access and collector tuning. There is + no Polymarket secret. +- Learners must source the file after every edit. + +## Common failures + +- Docker has no Server section: start Docker Desktop and enable WSL integration. +- Repo is under `/mnt/c`: reclone under `/home` before Docker builds. +- Cloud password was not saved: reset the default-user password in the Cloud console. +- Gamma blocked: proceed with Cloud checks and mark the learner for fixture mode. + +## Reset + +```bash +cd ~/ClickHouse_Demos/workshops/build_workshop/polymarket +cp -n .env.polymarket.example .env.polymarket +set -a; source ./.env.polymarket; set +a +./preflight.sh +``` + +Do not copy one learner's `.env.polymarket` to another machine. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/01-discover-markets.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/01-discover-markets.mdx new file mode 100644 index 0000000..3b49674 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/01-discover-markets.mdx @@ -0,0 +1,26 @@ +--- +title: 01 Discover markets · Instructor +description: Teach the condition/token distinction without drifting into trading. +--- + +## Timing + +10 minutes. + +## Talk track + +- Gamma discovers markets and metadata. +- One condition represents the market; each outcome has a CLOB token ID. +- WebSocket subscriptions use token IDs. Public trade reconciliation uses condition IDs. +- Prices between 0 and 1 are interpreted as indicative probabilities in this lesson. + +## Common failures + +- Empty trade response: the selected market is quiet, not broken. Try another condition. +- `JSONDecodeError`: venue proxy returned HTML. Confirm with `curl -I`, then use fixture mode. +- Learner asks about placing an order: out of scope. This track reads public data only. + +## Reset + +No resources are changed. Re-run the Gamma command or continue; the collector performs +its own discovery in Module 03. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/02-model-data.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/02-model-data.mdx new file mode 100644 index 0000000..a0f0ee0 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/02-model-data.mdx @@ -0,0 +1,33 @@ +--- +title: 02 Model the data · Instructor +description: Explain the schema choices and recover partial DDL runs. +--- + +## Timing + +20 minutes. + +## Talk track + +- Query patterns come before `ORDER BY`; the key cannot be casually changed later. +- Exact decimals avoid floating-point money/probability artifacts. +- Recent-time expressions lead the event keys; token/condition IDs follow for grouping. +- The incremental MV aggregates only BBO midpoint events. Trades remain separate. +- No partition is intentional; this track has no demonstrated retention lifecycle. + +## Common failures + +- Partial run: all statements use `IF NOT EXISTS`; rerun the complete blocks. +- MV exists but target does not: drop only the MV, rerun Step 2. +- Learner uses a different database: verify `.env.polymarket` says `polymarket`. + +## Reset + +For a disposable rehearsal only: + +```sql +DROP DATABASE IF EXISTS polymarket; +``` + +Then rerun both learner SQL blocks. Never run this against a learner's service without +their explicit confirmation. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/03-stream-live-data.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/03-stream-live-data.mdx new file mode 100644 index 0000000..c597dff --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/03-stream-live-data.mdx @@ -0,0 +1,32 @@ +--- +title: 03 Stream live data · Instructor +description: Run live, recognize safe degradation, and switch the room to fixture on time. +--- + +## Timing + +15 minutes. At minute 10, switch any blocked or quiet learner to fixture mode. + +## Talk track + +- The WebSocket gives immediacy, not durability. +- Data API trades reconcile with an overlap; deterministic IDs make reruns safe. +- CLOB REST keeps BBO/spread current while the socket reconnects. +- `degraded` is honest and useful. `unhealthy` means Cloud writes or all sources are stale. + +## Common failures + +- WebSocket handshake timeout: wait for `last_book_fallback_at`; do not restart-loop. +- Health 503: inspect `reason`, `queue_depth`, and `last_clickhouse_write_at`. +- Zero ticks after 60 seconds: use fixture so later modules are deterministic. +- Port 8090 occupied: use the troubleshooting port change. + +## Reset + +```bash +docker compose --env-file .env.polymarket down +docker compose --env-file .env.polymarket up -d --build collector +curl --fail --silent http://localhost:8090/health | python3 -m json.tool +``` + +Restarting hydrates recent IDs and trade checkpoints from ClickHouse Cloud. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/04-realtime-aggregates.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/04-realtime-aggregates.mdx new file mode 100644 index 0000000..f344715 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/04-realtime-aggregates.mdx @@ -0,0 +1,28 @@ +--- +title: 04 Real-time aggregates · Instructor +description: Make the incremental MV behavior and midpoint semantics explicit. +--- + +## Timing + +20 minutes. + +## Talk track + +- The MV processes inserted blocks; it does not rescan history on every dashboard refresh. +- `State` functions store partial aggregate states; matching `Merge` functions finalize them. +- OHLC is quote midpoint only. Mixing order-level change prices and last trades would make + the chart meaningless. +- The collector deduplicates before the MV sees rows; `ReplacingMergeTree` after the fact + would not undo aggregate double-counting. + +## Common failures + +- Aggregate empty: verify raw midpoint rows exist and the MV was created before ingestion. +- MV created late: restart fixture mode after creating it, or accept only new rows. +- Old newest minute: check collector health before debugging SQL. + +## Reset + +For a fresh aggregate without deleting raw data, recreate the target and MV, then allow +new events to populate it. Do not run `OPTIMIZE TABLE FINAL`. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/05-investigate-movement.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/05-investigate-movement.mdx new file mode 100644 index 0000000..79ecf80 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/05-investigate-movement.mdx @@ -0,0 +1,25 @@ +--- +title: 05 Investigate movement · Instructor +description: Keep the analysis evidence-based and explain empty windows. +--- + +## Timing + +20 minutes. + +## Talk track + +- Probability is the latest quote midpoint with an explicit freshness timestamp. +- A move is less persuasive when the spread is wide or the quote is stale. +- Volume velocity compares adjacent five-minute windows; it is not a forecast. +- Every statement is public-data analysis, not financial advice. + +## Common failures + +- Movers empty before minute five: expected; return after the other queries. +- Fixture prices show regular patterns: label them fixture data. +- A Yes/No pair does not sum to exactly 100: spreads and separate books explain the gap. + +## Reset + +No state change. Re-run the queries after enough time has accumulated. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/06-cloud-dashboard.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/06-cloud-dashboard.mdx new file mode 100644 index 0000000..8eb1120 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/06-cloud-dashboard.mdx @@ -0,0 +1,29 @@ +--- +title: 06 Cloud dashboard · Instructor +description: Verify the named saved queries, dashboard evidence, and permission fallback. +--- + +## Timing + +20 minutes. + +## Talk track + +- Dashboard panels are saved `SELECT` queries, not a second data pipeline. +- Use the exact five names so a room sweep is fast. +- The chart shows quote midpoint; the volume panel comes from reconciled trades. + +## Common failures + +- Learner cannot create dashboards: use the five saved SQL Console visualizations. +- Chart has no line: choose `minute` as time, probability as value, outcome as series. +- Stale chart: check the completion SQL and collector health, not browser refresh first. + +## Verification + +Ask learners to show either `Polymarket market pulse` with five views, or the five named +saved queries plus `age_seconds < 120` for fixture/active live data. + +## Reset + +Delete only the incomplete dashboard. Saved queries and Cloud data remain reusable. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/07-wrap-up.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/07-wrap-up.mdx new file mode 100644 index 0000000..d1e374f --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/07-wrap-up.mdx @@ -0,0 +1,30 @@ +--- +title: 07 Wrap up · Instructor +description: Capture proof, stop the room cleanly, and frame the production extension. +--- + +## Timing + +10 minutes. + +## Talk track + +- Ask one learner to explain the hybrid ingestion path and one to explain midpoint OHLC. +- Direct async writes fit this API-shaped workshop source. +- ClickPipes is the production choice when a durable supported stream already exists. +- Adding Kafka solely for the workshop would add setup, not learning. + +## Completion evidence + +Capture the final proof row and, if available, the dashboard. Record whether the room used +live, degraded-with-REST, or fixture mode. + +## Reset + +Every learner runs: + +```bash +docker compose --env-file .env.polymarket down +``` + +Cloud deletion is optional and belongs to each learner or organization owner. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/index.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/index.mdx new file mode 100644 index 0000000..f5dce1d --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/index.mdx @@ -0,0 +1,39 @@ +--- +title: Polymarket instructor track +description: A 2h10 run of show with a tested fixture fallback and objective completion checks. +--- + +## Before learners arrive + +- Confirm the workshop site is public and the Polymarket learner pages load without login. +- Verify Gamma, Data API, CLOB REST, and the WebSocket from the venue network. +- Prepare fixture mode even when live mode works; quiet markets are normal. +- Confirm each learner has a Cloud organization, an API key, and permission to create a + service, run SQL, save queries, and create dashboards. +- Do not ask learners to create a Polymarket account or wallet. + + + + Complete the [dev rehearsal](/docs/polymarket/rehearsal) from a fresh checkout before + requesting production promotion. + + + +## Run of show + +| Module | Budget | Hard stop | +|---|---:|---| +| 00 Setup | 15 min | Preflight ready; otherwise pair or provide a pre-created Cloud service | +| 01 Discover | 10 min | One condition and two token IDs understood | +| 02 Model | 20 min | Six objects returned by `SHOW TABLES` | +| 03 Stream | 15 min | Health acceptable and Cloud rows visible; switch to fixture at 10 min | +| 04 Aggregate | 20 min | Aggregate query and freshness check return | +| 05 Investigate | 20 min | Three immediate queries return; movers may wait | +| 06 Dashboard | 20 min | Dashboard or five saved-query fallback complete | +| 07 Wrap-up | 10 min | Final proof captured and local collector stopped | + +## Room rule + +`live`, `degraded` with fresh REST timestamps, and `fixture` are all valid teaching +states. `unhealthy` is not. State which one the room is using; never imply fixture data +is live market data. diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/instructor/meta.json b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/meta.json new file mode 100644 index 0000000..1ab1089 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/instructor/meta.json @@ -0,0 +1,16 @@ +{ + "title": "Polymarket · Instructor", + "description": "Timing, talk track, failure recovery, and reset steps.", + "icon": "Presentation", + "pages": [ + "index", + "00-setup", + "01-discover-markets", + "02-model-data", + "03-stream-live-data", + "04-realtime-aggregates", + "05-investigate-movement", + "06-cloud-dashboard", + "07-wrap-up" + ] +} diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/00-setup.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/00-setup.mdx new file mode 100644 index 0000000..b29a1fe --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/00-setup.mdx @@ -0,0 +1,182 @@ +--- +title: 00 Setup +description: Prepare macOS or WSL 2, create the ClickHouse Cloud service, and load one environment file. +--- + +Choose **macOS** or **Windows** in the page header. Windows commands run in Ubuntu on +WSL 2, not PowerShell, unless the page explicitly says PowerShell. + +## Outcome + +You have Docker, Git, Python, `clickhousectl`, the ClickHouse client, a Cloud service, +and a sourced `.env.polymarket`. The preflight prints `READY`. + +## Step 1 — Verify the laptop + + + +Install Docker Desktop for Mac. In Terminal run: + +```bash +docker version +docker compose version +git --version +python3 --version +``` + + + + + +The supported Windows environment is Ubuntu on WSL 2 with Docker Desktop WSL +integration. In Administrator PowerShell, install or verify it: + +```powershell +wsl --install -d Ubuntu +wsl --update +wsl --list --verbose +``` + +Restart if Windows asks. Enable **Docker Desktop -> Settings -> Resources -> WSL +Integration -> Ubuntu**. Then open Ubuntu and run: + +```bash +sudo apt-get update +sudo apt-get install -y ca-certificates curl git python3 +git config --global core.autocrlf input +docker version +docker compose version +git --version +python3 --version +pwd +``` + +Continue only if Ubuntu is WSL version 2, Docker shows Client and Server sections, and +`pwd` begins with `/home/`. Keep the repo under `/home`, not `/mnt/c`. + + + +## Step 2 — Clone the production workshop branch + + + +Run this from the Ubuntu terminal: + +```bash +cd ~ +``` + + + +```bash +git clone https://github.com/ClickHouse/ClickHouse_Demos.git +cd ClickHouse_Demos +git switch build-workshop-v1 +cd workshops/build_workshop/polymarket +cp .env.polymarket.example .env.polymarket +``` + +Run later commands from `ClickHouse_Demos/workshops/build_workshop/polymarket`. + +## Step 3 — Install `clickhousectl` and the ClickHouse client + +```bash +curl https://clickhouse.com/cli | sh +export PATH="$HOME/.local/bin:$PATH" +clickhousectl --version +clickhousectl local use stable +clickhouse client --version +``` + +`local use` installs the client binary only. It does not launch a database +server; every query targets ClickHouse Cloud. + +## Step 4 — Authenticate and create the Cloud service + +**In-person training:** use the learner-specific organization API key provided securely +by the trainer. + +The interactive login keeps the secret out of shell history: + +```bash +clickhousectl cloud auth login --interactive +clickhousectl cloud auth status +clickhousectl cloud org list +``` + +Trusted automation can use the explicit form: + +```bash +clickhousectl cloud auth login --api-key --api-secret +``` + +Create one service. Change the region only if the trainer specifies another: + +```bash +clickhousectl cloud service create \ + --name polymarket-workshop \ + --provider aws \ + --region ap-southeast-1 \ + --min-replica-memory-gb 8 \ + --max-replica-memory-gb 8 \ + --num-replicas 1 \ + --idle-scaling true \ + --idle-timeout-minutes 15 +``` + +Save the service ID and one-time default-user password. Wait until it is ready: + +```bash +clickhousectl cloud service list +clickhousectl cloud service get +``` + +## Step 5 — Fill and source `.env.polymarket` + +Open the service **Connect** dialog. Put the bare hostname and password into +`.env.polymarket`; do not include `https://` or `:8443` in the host. + +```bash +${EDITOR:-vi} .env.polymarket +set -a; source ./.env.polymarket; set +a +``` + +Confirm that the shell loaded the non-secret values: + +```bash +printf 'host=%s database=%s mode=%s\n' \ + "$CLICKHOUSE_HOST" "$CLICKHOUSE_DATABASE" "$POLYMARKET_MODE" +``` + +Expected: the Cloud hostname, `polymarket`, and `live`. + +## Step 6 — Run preflight + +```bash +chmod +x ./preflight.sh +./preflight.sh +``` + +Expected: + +```text +READY: Docker, ClickHouse Cloud, Gamma, Data API, and CLOB REST are reachable +``` + +In fixture mode on a network that blocks Polymarket, the final line is instead: + +```text +READY: Docker and ClickHouse Cloud are reachable; fixture mode will supply data +``` + +If only the Polymarket API check fails, use the fixture fallback in +[Troubleshooting](/docs/polymarket/learner/troubleshooting#the-classroom-network-blocks-polymarket). + +## Done when + +- `clickhouse client --version` succeeds; +- `clickhousectl cloud service get ` reports the service ready; +- `printf` shows the expected host/database/mode; and +- `./preflight.sh` prints `READY`. + +Next: [discover the markets you will watch](/docs/polymarket/learner/01-discover-markets). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/01-discover-markets.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/01-discover-markets.mdx new file mode 100644 index 0000000..0993f0d --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/01-discover-markets.mdx @@ -0,0 +1,78 @@ +--- +title: 01 Discover markets +description: Inspect public active markets, outcomes, condition IDs, and CLOB token IDs. +--- + +## Starting point + +You are in `workshops/build_workshop/polymarket`, `.env.polymarket` is sourced, and the +preflight is ready. + +## Why + +Polymarket uses a condition ID for a market and a separate token ID for each outcome. +The WebSocket subscribes by token ID; the public trades API filters by condition ID. + +## Step 1 — Ask Gamma for the five busiest active markets + +```bash +curl --fail --silent --show-error --get \ + 'https://gamma-api.polymarket.com/markets' \ + --data-urlencode 'active=true' \ + --data-urlencode 'closed=false' \ + --data-urlencode 'limit=20' \ + --data-urlencode 'order=volume24hr' \ + --data-urlencode 'ascending=false' \ + | python3 -c ' +import json, sys +selected = 0 +for market in json.load(sys.stdin): + if not market.get("acceptingOrders"): + continue + outcomes = json.loads(market["outcomes"]) + tokens = json.loads(market["clobTokenIds"]) + print(market["question"]) + print(" condition:", market["conditionId"]) + for outcome, token in zip(outcomes, tokens): + print(f" {outcome}: {token}") + selected += 1 + if selected == 5: + break +' +``` + +Expected: five questions. Each prints one 66-character condition hash and one token ID +per outcome. The exact questions change with the live market. + +## Step 2 — Confirm public trades for one condition + +Copy one condition ID from the output: + +```bash +export POLYMARKET_CONDITION_ID='0x...' +curl --fail --silent --show-error --get \ + 'https://data-api.polymarket.com/trades' \ + --data-urlencode "market=$POLYMARKET_CONDITION_ID" \ + --data-urlencode 'limit=3' \ + | python3 -c ' +import json, sys +for trade in json.load(sys.stdin): + print(trade["outcome"], trade["side"], trade["size"], "@", trade["price"]) +' +``` + +An empty response is valid for a quiet market. Pick another condition if you want to see +trade rows immediately. No authentication is used. + +## Step 3 — Understand the collector selection + +The collector repeats Step 1 at startup, selects five markets, parses the JSON-encoded +outcome/token arrays, and stores both outcome rows in `polymarket.markets`. You do not +hard-code today's token IDs into `.env.polymarket`. + +## Done when + +- you can identify the condition ID and both outcome token IDs for one market; and +- the public trades command returns JSON, even if the array is empty. + +Next: [create the ClickHouse data model](/docs/polymarket/learner/02-model-data). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/02-model-data.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/02-model-data.mdx new file mode 100644 index 0000000..78c9542 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/02-model-data.mdx @@ -0,0 +1,162 @@ +--- +title: 02 Model the data +description: Create typed market, tick, trade, and one-minute aggregate tables in ClickHouse Cloud. +--- + +## Starting point + +`.env.polymarket` is sourced and you know why condition IDs and token IDs are different. + +## Why these tables + +The five queries later read recent time windows across every watched market. Event-table +keys therefore start with an hour/time key, followed by the token or condition used for +grouping. Known fields use native types: `UInt256` for token IDs, +`DateTime64` for event time, exact decimals for prices and sizes, and enums for bounded +event values. The opaque source payload stays a string because no query reads its fields. + +There is no partition key. This short-lived workshop has no proven retention boundary; +adding partitions before a lifecycle requirement would create small parts without a +benefit. + +## Step 1 — Create the database and raw tables + +Copy the whole block into the ClickHouse Cloud SQL console and run it: + +```sql +CREATE DATABASE IF NOT EXISTS polymarket; + +CREATE TABLE IF NOT EXISTS polymarket.markets +( + market_id UInt64, + condition_id FixedString(66), + token_id UInt256, + outcome LowCardinality(String), + question String, + slug String, + active Bool, + accepting_orders Bool, + volume_24h Decimal128(8), + observed_at DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(observed_at) +ORDER BY (condition_id, token_id); + +CREATE TABLE IF NOT EXISTS polymarket.price_ticks +( + event_id FixedString(64), + condition_id FixedString(66), + token_id UInt256, + event_at DateTime64(3, 'UTC'), + observed_at DateTime64(3, 'UTC'), + event_kind Enum8( + 'book_snapshot' = 1, + 'price_change' = 2, + 'last_trade_price' = 3, + 'best_bid_ask' = 4, + 'rest_book' = 5 + ), + source Enum8('WEBSOCKET' = 1, 'CLOB_REST' = 2, 'FIXTURE' = 3), + price Decimal64(12), + size Decimal128(8), + side Enum8('UNKNOWN' = 0, 'BUY' = 1, 'SELL' = 2), + best_bid Decimal64(12), + best_ask Decimal64(12), + midpoint Decimal64(12), + source_hash String, + raw_payload String +) +ENGINE = MergeTree +ORDER BY (toStartOfHour(event_at), token_id, event_at, event_id); + +CREATE TABLE IF NOT EXISTS polymarket.trades +( + trade_id FixedString(64), + condition_id FixedString(66), + token_id UInt256, + event_at DateTime64(3, 'UTC'), + observed_at DateTime64(3, 'UTC'), + proxy_wallet FixedString(42), + side Enum8('UNKNOWN' = 0, 'BUY' = 1, 'SELL' = 2), + price Decimal64(12), + size Decimal128(8), + outcome LowCardinality(String), + transaction_hash FixedString(66), + title String +) +ENGINE = ReplacingMergeTree(observed_at) +ORDER BY (toStartOfHour(event_at), condition_id, event_at, trade_id); + +CREATE OR REPLACE VIEW polymarket.trades_clean AS +SELECT * +FROM polymarket.trades FINAL; +``` + +The collector prevents duplicates before insert. `ReplacingMergeTree` is a second safety +net. The `trades_clean` view makes the small workshop queries deterministic while merges +are still in progress. + +## Step 2 — Create the one-minute midpoint aggregate + +```sql +CREATE TABLE IF NOT EXISTS polymarket.market_midpoints_1m +( + token_id UInt256, + minute DateTime('UTC'), + open AggregateFunction(argMin, Decimal64(12), Tuple(DateTime64(3, 'UTC'), FixedString(64))), + high AggregateFunction(max, Decimal64(12)), + low AggregateFunction(min, Decimal64(12)), + close AggregateFunction(argMax, Decimal64(12), Tuple(DateTime64(3, 'UTC'), FixedString(64))), + updates AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (minute, token_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS polymarket.market_midpoints_1m_mv +TO polymarket.market_midpoints_1m +AS +SELECT + token_id, + toStartOfMinute(event_at) AS minute, + argMinState(midpoint, tuple(event_at, event_id)) AS open, + maxState(midpoint) AS high, + minState(midpoint) AS low, + argMaxState(midpoint, tuple(event_at, event_id)) AS close, + countState() AS updates +FROM polymarket.price_ticks +WHERE midpoint > 0 + AND event_kind IN ('book_snapshot', 'price_change', 'best_bid_ask', 'rest_book') +GROUP BY token_id, minute; +``` + +This materialized view aggregates only quote midpoints. It deliberately excludes the +changed order-level price and last-trade price, so the OHLC series has one meaning. + +## Step 3 — Verify every object + +```bash +clickhouse client \ + --host "$CLICKHOUSE_HOST" \ + --port "$CLICKHOUSE_PORT" \ + --user "$CLICKHOUSE_USER" \ + --password "$CLICKHOUSE_PASSWORD" \ + --secure \ + --query "SHOW TABLES FROM polymarket" +``` + +Expected names include: + +```text +market_midpoints_1m +market_midpoints_1m_mv +markets +price_ticks +trades +trades_clean +``` + +## Done when + +`SHOW TABLES` returns all six objects with no local ClickHouse server running. + +Next: [start the live collector](/docs/polymarket/learner/03-stream-live-data). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/03-stream-live-data.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/03-stream-live-data.mdx new file mode 100644 index 0000000..37c9bdb --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/03-stream-live-data.mdx @@ -0,0 +1,109 @@ +--- +title: 03 Stream live data +description: Run the resilient collector and verify WebSocket, REST reconciliation, and Cloud writes. +--- + +## Starting point + +The six `polymarket` objects exist and `.env.polymarket` is sourced. + +## What starts + +One stateless Python container: + +- discovers five active markets through Gamma; +- subscribes to both outcome tokens on the public CLOB WebSocket; +- reconciles public trades every 10 seconds; +- polls CLOB books when the WebSocket is stalled; and +- writes acknowledged async inserts to ClickHouse Cloud. + +There is no local database, broker, dashboard server, or Polymarket credential. + +## Step 1 — Build and start the collector + +```bash +docker compose --env-file .env.polymarket up -d --build collector +docker compose --env-file .env.polymarket ps +``` + +The status becomes `healthy` after discovery and the first successful Cloud write. A +`degraded` application status is still Docker-healthy when REST is current and the +WebSocket is reconnecting. + +## Step 2 — Read the health contract + +```bash +curl --fail --silent http://localhost:8090/health \ + | python3 -m json.tool +``` + +Expected fields: + +```json +{ + "status": "live", + "websocket": "connected", + "queue_depth": 0, + "queue_capacity": 10000, + "watched_markets": 5, + "watched_tokens": 10, + "fresh_tokens": 10 +} +``` + +`status: degraded` with `reason: websocket_stale_rest_active` is acceptable if +`last_trade_reconcile_at` and `last_book_fallback_at` keep advancing. `unhealthy` is not +acceptable; use Troubleshooting. + +## Step 3 — Watch source and write events + +```bash +docker compose --env-file .env.polymarket logs --tail=30 collector +``` + +Logs are JSON. Look for `collector_ready`. Source or ClickHouse failures include a +bounded error preview and retry delay; no password is logged. + +## Step 4 — Prove rows are in Cloud + +```bash +clickhouse client \ + --host "$CLICKHOUSE_HOST" \ + --port "$CLICKHOUSE_PORT" \ + --user "$CLICKHOUSE_USER" \ + --password "$CLICKHOUSE_PASSWORD" \ + --secure \ + --query " + SELECT 'markets' AS table, count() AS rows FROM polymarket.markets + UNION ALL + SELECT 'quote_midpoints', countIf(midpoint > 0) FROM polymarket.price_ticks + UNION ALL + SELECT 'trades', count() FROM polymarket.trades_clean + UNION ALL + SELECT 'one_minute_states', count() FROM polymarket.market_midpoints_1m + " +``` + +`markets`, `quote_midpoints`, and `one_minute_states` must be greater than zero before +Module 04. `trades` normally grows within a minute; a quiet market may delay it. + +## Step 5 — Use deterministic fixture mode only when needed + +If the room network blocks Polymarket or no selected market moves after 60 seconds: + +```bash +sed -i.bak 's/^POLYMARKET_MODE=.*/POLYMARKET_MODE=fixture/' .env.polymarket +set -a; source ./.env.polymarket; set +a +docker compose --env-file .env.polymarket up -d --build --force-recreate collector +``` + +Run the health and row-count checks again. Expected status: `fixture`; tick and trade +counts increase every five seconds. Keep the backup file until the module ends. + +## Done when + +- health is `live`, `degraded` with fresh REST timestamps, or `fixture`; +- `watched_markets` is 5; and +- all four Cloud row counts return, with quote and one-minute rows greater than zero. + +Next: [query the incremental aggregate](/docs/polymarket/learner/04-realtime-aggregates). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/04-realtime-aggregates.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/04-realtime-aggregates.mdx new file mode 100644 index 0000000..ca9cc8b --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/04-realtime-aggregates.mdx @@ -0,0 +1,82 @@ +--- +title: 04 Real-time aggregates +description: Read one-minute quote-midpoint OHLC maintained by an incremental materialized view. +--- + +## Starting point + +The collector is healthy and `polymarket.price_ticks` contains rows. + +## Why + +Dashboard users repeatedly ask for the same one-minute series. Computing it once as new +blocks arrive shifts work from every dashboard refresh to insert time. The raw table +remains available for ad-hoc questions. + +## Step 1 — Query the aggregate states correctly + +```sql +SELECT + minute, + token_id, + round(argMinMerge(open) * 100, 2) AS open_percent, + round(maxMerge(high) * 100, 2) AS high_percent, + round(minMerge(low) * 100, 2) AS low_percent, + round(argMaxMerge(close) * 100, 2) AS close_percent, + countMerge(updates) AS updates +FROM polymarket.market_midpoints_1m +WHERE minute >= now() - INTERVAL 30 MINUTE +GROUP BY minute, token_id +ORDER BY minute DESC, token_id +LIMIT 30; +``` + +`argMinState`/`argMaxState` were written by the view; the query finalizes them with the +matching `Merge` functions. Open and close use event time plus deterministic event ID, +so out-of-order arrivals and same-millisecond ties settle consistently. + +## Step 2 — Compare rows read by raw and aggregate queries + +Run the raw equivalent: + +```sql +SELECT + toStartOfMinute(event_at) AS minute, + token_id, + round(argMin(midpoint, tuple(event_at, event_id)) * 100, 2) AS open_percent, + round(max(midpoint) * 100, 2) AS high_percent, + round(min(midpoint) * 100, 2) AS low_percent, + round(argMax(midpoint, tuple(event_at, event_id)) * 100, 2) AS close_percent, + count() AS updates +FROM polymarket.price_ticks +WHERE midpoint > 0 + AND event_at >= now() - INTERVAL 30 MINUTE + AND event_kind IN ('book_snapshot', 'price_change', 'best_bid_ask', 'rest_book') +GROUP BY minute, token_id +ORDER BY minute DESC, token_id +LIMIT 30; +``` + +In the SQL console, compare **read rows** for both queries. The aggregate reads block-level +states that `AggregatingMergeTree` combines in the background, usually far fewer rows +than scanning every source update. + +## Step 3 — Verify the view is insert-driven + +```sql +SELECT + max(minute) AS newest_minute, + dateDiff('second', newest_minute, now()) AS age_seconds, + countMerge(updates) AS source_updates +FROM polymarket.market_midpoints_1m; +``` + +In live or fixture mode, `newest_minute` advances without a scheduled refresh job. + +## Done when + +- the aggregate query returns OHLC rows; +- open/high/low/close are probabilities between 0 and 100; and +- `newest_minute` is current for an active feed. + +Next: [investigate a market move](/docs/polymarket/learner/05-investigate-movement). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/05-investigate-movement.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/05-investigate-movement.mdx new file mode 100644 index 0000000..4e57fdb --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/05-investigate-movement.mdx @@ -0,0 +1,117 @@ +--- +title: 05 Investigate movement +description: Answer four operational market questions with explicit ClickHouse SQL. +--- + +## Starting point + +The raw and one-minute tables contain current data. + +## Question 1 — What is the current probability? + +```sql +SELECT + m.token_id, + m.question, + m.outcome, + round(argMax(t.midpoint, t.event_at) * 100, 2) AS probability_percent, + max(t.event_at) AS last_update +FROM polymarket.price_ticks AS t +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = t.token_id +WHERE t.midpoint > 0 + AND t.event_at >= now() - INTERVAL 30 MINUTE +GROUP BY m.token_id, m.question, m.outcome +ORDER BY m.question, m.outcome; +``` + +The midpoint is an indicative probability from the best bid and ask, not a promise of a +tradeable price. + +## Question 2 — Which outcome moved most? + +```sql +WITH now() AS current_time +SELECT + m.token_id, + m.question, + m.outcome, + round(argMaxIf(t.midpoint, t.event_at, t.event_at > current_time - INTERVAL 1 MINUTE) * 100, 2) AS now_percent, + round(argMaxIf(t.midpoint, t.event_at, t.event_at <= current_time - INTERVAL 5 MINUTE) * 100, 2) AS five_minutes_ago_percent, + round(now_percent - five_minutes_ago_percent, 2) AS move_points +FROM polymarket.price_ticks AS t +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = t.token_id +WHERE t.midpoint > 0 + AND t.event_at >= current_time - INTERVAL 15 MINUTE +GROUP BY m.token_id, m.question, m.outcome +HAVING now_percent > 0 AND five_minutes_ago_percent > 0 +ORDER BY abs(move_points) DESC; +``` + +If this is empty, the feed has not accumulated five minutes. Continue with the next +queries and return later. + +## Question 3 — Is the spread wide or the data stale? + +```sql +SELECT + m.token_id, + m.question, + m.outcome, + round(argMax(t.best_bid, t.event_at) * 100, 2) AS bid_percent, + round(argMax(t.best_ask, t.event_at) * 100, 2) AS ask_percent, + round(ask_percent - bid_percent, 2) AS spread_points, + dateDiff('second', max(t.event_at), now()) AS age_seconds +FROM polymarket.price_ticks AS t +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = t.token_id +WHERE t.best_bid > 0 + AND t.best_ask > 0 + AND t.event_at >= now() - INTERVAL 30 MINUTE +GROUP BY m.token_id, m.question, m.outcome +ORDER BY spread_points DESC; +``` + +A move with a wide spread or old quote deserves less confidence than a fresh, tight +market. + +## Question 4 — Did recent trade volume accelerate? + +```sql +SELECT + condition_id, + token_id, + title, + outcome, + round(sumIf(price * size, event_at >= now() - INTERVAL 5 MINUTE), 2) AS current_5m_usd, + round(sumIf( + price * size, + event_at >= now() - INTERVAL 10 MINUTE + AND event_at < now() - INTERVAL 5 MINUTE + ), 2) AS previous_5m_usd, + round(current_5m_usd / greatest(previous_5m_usd, 0.01), 2) AS velocity_ratio +FROM polymarket.trades_clean +WHERE event_at >= now() - INTERVAL 10 MINUTE +GROUP BY condition_id, token_id, title, outcome +ORDER BY current_5m_usd DESC; +``` + +This is public matched volume represented as `price * size`; it is analysis, not a +recommendation. + +## Done when + +At least the current probability, spread/freshness, and volume queries return without +error. After five minutes, the movers query should also return rows. + +Next: [publish the Cloud dashboard](/docs/polymarket/learner/06-cloud-dashboard). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/06-cloud-dashboard.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/06-cloud-dashboard.mdx new file mode 100644 index 0000000..2212c42 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/06-cloud-dashboard.mdx @@ -0,0 +1,93 @@ +--- +title: 06 Cloud dashboard +description: Save the investigation queries and assemble the Polymarket market pulse dashboard. +--- + +## Starting point + +The four Module 05 queries run in the ClickHouse Cloud SQL console. Your Cloud role can +save queries and create dashboards. + +## Goal + +One dashboard named **Polymarket market pulse** with four current views and a one-minute +line chart. No local dashboard product is required. + +## Step 1 — Save the four investigation queries + +In the SQL console, run each Module 05 block and use **Save** with these exact names: + +1. `Current probability` +2. `Five-minute movers` +3. `Spread and freshness` +4. `Volume velocity` + +Do not change the SQL while saving; the instructor uses the names to verify the room. + +## Step 2 — Save the one-minute chart query + +```sql +SELECT + c.minute, + m.token_id, + m.question, + m.outcome, + round(argMaxMerge(c.close) * 100, 2) AS probability_percent +FROM polymarket.market_midpoints_1m AS c +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = c.token_id +WHERE c.minute >= now() - INTERVAL 30 MINUTE +GROUP BY c.minute, m.token_id, m.question, m.outcome +ORDER BY c.minute, m.token_id; +``` + +Save it as `One-minute probability` and choose a line visualization. Use `minute` as the +time axis, `probability_percent` as the value, and question/outcome/token ID as the +series so identically worded markets stay separate. + +## Step 3 — Assemble the dashboard + +1. Open **Dashboards** in the ClickHouse Cloud console. +2. Create `Polymarket market pulse`. +3. Add `Current probability` as a table or number view. +4. Add `Five-minute movers` as a table sorted by absolute movement. +5. Add `Spread and freshness` as a table. +6. Add `Volume velocity` as a table or bar chart. +7. Add `One-minute probability` as the line chart. +8. Set the dashboard refresh to the shortest option available in your console. + +## Step 4 — Verify objective evidence + +The dashboard is complete when: + +- at least one watched market appears; +- probability, bid, and ask values are not null; +- the freshness age is visible; and +- the line chart advances after the next live or fixture event. + +## If Dashboards is unavailable + +Keep the five saved queries and their SQL Console visualizations. Run this completion +check: + +```sql +SELECT + count() AS watched_outcomes, + max(event_at) AS newest_tick, + dateDiff('second', newest_tick, now()) AS age_seconds +FROM polymarket.price_ticks +WHERE midpoint > 0; +``` + +Completion fallback: `watched_outcomes > 0` and `age_seconds < 120` in fixture mode or +an active live feed. This is the only fallback; do not install a local dashboard. + +## Done when + +The dashboard has five views, or all five saved SQL Console visualizations exist and the +completion query is fresh. + +Next: [wrap up and clean up](/docs/polymarket/learner/07-wrap-up). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/07-wrap-up.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/07-wrap-up.mdx new file mode 100644 index 0000000..047ff53 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/07-wrap-up.mdx @@ -0,0 +1,57 @@ +--- +title: 07 Wrap up +description: Prove the track is complete, stop local work, and understand the production scale-up path. +--- + +## Final proof + +```sql +SELECT + (SELECT uniqExact(condition_id) FROM polymarket.markets FINAL) AS markets, + (SELECT count() FROM polymarket.price_ticks) AS ticks, + (SELECT count() FROM polymarket.trades_clean) AS trades, + (SELECT count() FROM polymarket.market_midpoints_1m) AS midpoint_minutes, + (SELECT max(event_at) FROM polymarket.price_ticks) AS newest_tick; +``` + +You are done when `markets`, `ticks`, and `midpoint_minutes` are greater than zero and +`newest_tick` matches this run. A quiet live feed can legitimately have zero reconciled +trades; fixture mode always produces trades. + +## What you built + +- public market discovery with no credentials; +- a live WebSocket path with heartbeat, stall detection, reconnect, and REST BBO fallback; +- a reconciled public trade path with overlapping windows and deterministic IDs; +- typed ClickHouse tables designed around actual filters; +- an insert-time one-minute quote-midpoint aggregate; and +- a Cloud dashboard or equivalent saved SQL visualizations. + +## Stop the local collector + +```bash +docker compose --env-file .env.polymarket down +``` + +This removes the stateless container. Cloud data and saved queries remain. + +## Optional Cloud cleanup + +Only run this if you no longer want the workshop data: + +```sql +DROP DATABASE polymarket; +``` + +Delete or idle the Cloud service through `clickhousectl` or the console according to +your organization policy. + +## How this scales in production + +The workshop writes directly because the source is an HTTP/WebSocket API and the volume +is small. If a production relay already publishes to Kafka, Kinesis, Pub/Sub, or another +supported stream, use ClickPipes for managed offsets, schema mapping, backpressure, and +error handling. Do not add a broker only to make the workshop look more distributed. + +Return to the [workshop catalog](/docs) or compare this path with the +[AI SRE track](/docs/ai-sre). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/index.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/index.mdx new file mode 100644 index 0000000..8987f99 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/index.mdx @@ -0,0 +1,27 @@ +--- +title: Polymarket learner track +description: A two-hour path from public live data to a ClickHouse Cloud market pulse dashboard. +--- + +Work the modules in order. Each page states its input, exact commands, expected result, +and completion check. + +## Progress + +0. [Setup](/docs/polymarket/learner/00-setup) — prepare macOS or WSL 2, create the Cloud service, and load the environment. +1. [Discover markets](/docs/polymarket/learner/01-discover-markets) — inspect active markets, outcomes, condition IDs, and token IDs. +2. [Model the data](/docs/polymarket/learner/02-model-data) — create typed tables and the one-minute midpoint materialized view. +3. [Stream live data](/docs/polymarket/learner/03-stream-live-data) — run the collector and verify live or degraded-but-current ingestion. +4. [Real-time aggregates](/docs/polymarket/learner/04-realtime-aggregates) — read quote-midpoint OHLC from an incremental aggregate. +5. [Investigate movement](/docs/polymarket/learner/05-investigate-movement) — query movers, spread, freshness, and volume velocity. +6. [Cloud dashboard](/docs/polymarket/learner/06-cloud-dashboard) — save the queries and publish the market pulse dashboard. +7. [Wrap up](/docs/polymarket/learner/07-wrap-up) — prove completion, stop the collector, and understand the Kafka/ClickPipes scale-up path. + +## Guardrails + +- Public market data only. No wallet, trading account, API secret, order, or financial advice. +- ClickHouse Cloud is the only database. Docker runs one stateless collector. +- Live mode is preferred. Fixture mode is an explicit fallback for a blocked or quiet feed. +- Never commit `.env.polymarket`. + +Start with [Module 00](/docs/polymarket/learner/00-setup). diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/meta.json b/workshops/build_workshop/playbook/content/docs/polymarket/learner/meta.json new file mode 100644 index 0000000..6495774 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/meta.json @@ -0,0 +1,17 @@ +{ + "title": "Polymarket · Learner", + "description": "Build the live market-data track step by step.", + "icon": "GraduationCap", + "pages": [ + "index", + "00-setup", + "01-discover-markets", + "02-model-data", + "03-stream-live-data", + "04-realtime-aggregates", + "05-investigate-movement", + "06-cloud-dashboard", + "07-wrap-up", + "troubleshooting" + ] +} diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/learner/troubleshooting.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/learner/troubleshooting.mdx new file mode 100644 index 0000000..050b1b6 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/learner/troubleshooting.mdx @@ -0,0 +1,81 @@ +--- +title: Polymarket troubleshooting +description: Exact recovery for setup, source, collector, and ClickHouse failures. +--- + +## The classroom network blocks Polymarket + +Symptom: preflight cannot reach Gamma, or health stays `degraded` with source errors. + +```bash +sed -i.bak 's/^POLYMARKET_MODE=.*/POLYMARKET_MODE=fixture/' .env.polymarket +set -a; source ./.env.polymarket; set +a +docker compose --env-file .env.polymarket up -d --build --force-recreate collector +curl --fail --silent http://localhost:8090/health | python3 -m json.tool +``` + +Expected: `status` is `fixture`; data grows every five seconds. + +## Health says `websocket_stale_rest_active` + +This is recoverable when `last_trade_reconcile_at` and `last_book_fallback_at` keep +advancing. The collector reconnects with backoff. Do not restart it repeatedly. + +```bash +docker compose --env-file .env.polymarket logs --tail=50 collector +sleep 35 +curl --fail --silent http://localhost:8090/health | python3 -m json.tool +``` + +Use fixture mode only if both REST timestamps remain null or stale. + +## Health says `clickhouse_write_retrying` or `clickhouse_write_stalled` + +While the process is running, the exact batch remains in memory and retries with the same +deduplication token. Fix the Cloud values, source the file, then recreate the collector: + +```bash +${EDITOR:-vi} .env.polymarket +set -a; source ./.env.polymarket; set +a +./preflight.sh +docker compose --env-file .env.polymarket up -d --force-recreate collector +``` + +Do not drop the tables; a retry is safe. + +A forced recreate discards any quote ticks that were still only in memory. The restarted +collector repopulates current books through REST and reconciles public trades from the +last acknowledged checkpoint. The workshop does not claim a durable local queue. + +## `missing ClickHouse tables` appears + +Module 02 was skipped or ran against another service. Source the current file and check: + +```bash +set -a; source ./.env.polymarket; set +a +clickhouse client \ + --host "$CLICKHOUSE_HOST" \ + --port "$CLICKHOUSE_PORT" \ + --user "$CLICKHOUSE_USER" \ + --password "$CLICKHOUSE_PASSWORD" \ + --secure \ + --query "SHOW TABLES FROM polymarket" +``` + +Return to Module 02 if an object is missing. + +## The mover query is empty + +It requires observations on both sides of a five-minute boundary. Wait until the +collector has run for at least six minutes, or use fixture mode for a timed class. + +## Port 8090 is already in use + +Change `POLYMARKET_HEALTH_PORT` in `.env.polymarket`, source it, and recreate: + +```bash +sed -i.bak 's/^POLYMARKET_HEALTH_PORT=.*/POLYMARKET_HEALTH_PORT=8091/' .env.polymarket +set -a; source ./.env.polymarket; set +a +docker compose --env-file .env.polymarket up -d --force-recreate collector +curl --fail --silent http://localhost:8091/health | python3 -m json.tool +``` diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/meta.json b/workshops/build_workshop/playbook/content/docs/polymarket/meta.json new file mode 100644 index 0000000..34cacc9 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/meta.json @@ -0,0 +1,6 @@ +{ + "title": "Polymarket track", + "description": "Live public prediction-market analytics on ClickHouse Cloud.", + "icon": "ChartCandlestick", + "pages": ["index", "learner", "instructor"] +} diff --git a/workshops/build_workshop/playbook/content/docs/polymarket/rehearsal.mdx b/workshops/build_workshop/playbook/content/docs/polymarket/rehearsal.mdx new file mode 100644 index 0000000..3f36175 --- /dev/null +++ b/workshops/build_workshop/playbook/content/docs/polymarket/rehearsal.mdx @@ -0,0 +1,25 @@ +--- +title: Polymarket dev rehearsal +description: Maintainer-only staging run before production promotion. +--- + + + This page is for maintainers validating the staging branch. Public learners use + `build-workshop-v1` from Module 00 after production promotion. + + +Use a fresh checkout so the rehearsal proves distribution, not only a local worktree: + +```bash +git clone https://github.com/ClickHouse/ClickHouse_Demos.git ClickHouse_Demos-polymarket-rehearsal +cd ClickHouse_Demos-polymarket-rehearsal +git switch dev-build-workshop-v1 +cd workshops/build_workshop/polymarket +cp .env.polymarket.example .env.polymarket +``` + +Continue Module 00 at **Step 3** using this staging checkout; do not repeat its production +clone/switch step. Complete the remaining Modules 00–07, including one forced WebSocket +failure or venue-blocked test, one fixture-mode run, collector restart, and cleanup. +Record elapsed time and any command that required explanation not present on the learner +page. diff --git a/workshops/build_workshop/playbook/src/app/docs/[[...slug]]/page.tsx b/workshops/build_workshop/playbook/src/app/docs/[[...slug]]/page.tsx index d6d73d5..cdcbdc8 100644 --- a/workshops/build_workshop/playbook/src/app/docs/[[...slug]]/page.tsx +++ b/workshops/build_workshop/playbook/src/app/docs/[[...slug]]/page.tsx @@ -1,4 +1,4 @@ -import { getPageImage, getPageMarkdownUrl, source } from '@/lib/source'; +import { getPageImage, getPageMarkdownUrl, isHiddenRehearsal, source } from '@/lib/source'; import { DocsBody, DocsDescription, @@ -16,6 +16,7 @@ import { PlatformSelector, PlatformShellNote } from '@/components/platform'; export default async function Page(props: PageProps<'/docs/[[...slug]]'>) { const params = await props.params; + if (isHiddenRehearsal(params.slug)) notFound(); const page = source.getPage(params.slug); if (!page) notFound(); @@ -31,7 +32,7 @@ export default async function Page(props: PageProps<'/docs/[[...slug]]'>) { @@ -50,11 +51,12 @@ export default async function Page(props: PageProps<'/docs/[[...slug]]'>) { } export async function generateStaticParams() { - return source.generateParams(); + return source.generateParams().filter((params) => !isHiddenRehearsal(params.slug)); } export async function generateMetadata(props: PageProps<'/docs/[[...slug]]'>): Promise { const params = await props.params; + if (isHiddenRehearsal(params.slug)) notFound(); const page = source.getPage(params.slug); if (!page) notFound(); diff --git a/workshops/build_workshop/playbook/src/app/llms.mdx/docs/[[...slug]]/route.ts b/workshops/build_workshop/playbook/src/app/llms.mdx/docs/[[...slug]]/route.ts index 250181a..04837ba 100644 --- a/workshops/build_workshop/playbook/src/app/llms.mdx/docs/[[...slug]]/route.ts +++ b/workshops/build_workshop/playbook/src/app/llms.mdx/docs/[[...slug]]/route.ts @@ -1,11 +1,13 @@ -import { getLLMText, getPageMarkdownUrl, source } from '@/lib/source'; +import { getLLMText, getPageMarkdownUrl, isHiddenRehearsal, source } from '@/lib/source'; import { notFound } from 'next/navigation'; export const revalidate = false; export async function GET(_req: Request, { params }: RouteContext<'/llms.mdx/docs/[[...slug]]'>) { const { slug } = await params; - const page = source.getPage(slug?.slice(0, -1)); + const pageSlugs = slug?.slice(0, -1); + if (isHiddenRehearsal(pageSlugs)) notFound(); + const page = source.getPage(pageSlugs); if (!page) notFound(); return new Response(await getLLMText(page), { @@ -16,7 +18,7 @@ export async function GET(_req: Request, { params }: RouteContext<'/llms.mdx/doc } export function generateStaticParams() { - return source.getPages().map((page) => ({ + return source.getPages().filter((page) => !isHiddenRehearsal(page.slugs)).map((page) => ({ lang: page.locale, slug: getPageMarkdownUrl(page).segments, })); diff --git a/workshops/build_workshop/playbook/src/app/og/docs/[...slug]/route.tsx b/workshops/build_workshop/playbook/src/app/og/docs/[...slug]/route.tsx index 877166d..2406b56 100644 --- a/workshops/build_workshop/playbook/src/app/og/docs/[...slug]/route.tsx +++ b/workshops/build_workshop/playbook/src/app/og/docs/[...slug]/route.tsx @@ -1,4 +1,4 @@ -import { getPageImage, source } from '@/lib/source'; +import { getPageImage, isHiddenRehearsal, source } from '@/lib/source'; import { notFound } from 'next/navigation'; import { ImageResponse } from 'next/og'; import { generate as DefaultImage } from 'fumadocs-ui/og'; @@ -8,7 +8,9 @@ export const revalidate = false; export async function GET(_req: Request, { params }: RouteContext<'/og/docs/[...slug]'>) { const { slug } = await params; - const page = source.getPage(slug.slice(0, -1)); + const pageSlugs = slug.slice(0, -1); + if (isHiddenRehearsal(pageSlugs)) notFound(); + const page = source.getPage(pageSlugs); if (!page) notFound(); return new ImageResponse( @@ -21,7 +23,7 @@ export async function GET(_req: Request, { params }: RouteContext<'/og/docs/[... } export function generateStaticParams() { - return source.getPages().map((page) => ({ + return source.getPages().filter((page) => !isHiddenRehearsal(page.slugs)).map((page) => ({ lang: page.locale, slug: getPageImage(page).segments, })); diff --git a/workshops/build_workshop/playbook/src/components/mdx.tsx b/workshops/build_workshop/playbook/src/components/mdx.tsx index 550da23..94d33ea 100644 --- a/workshops/build_workshop/playbook/src/components/mdx.tsx +++ b/workshops/build_workshop/playbook/src/components/mdx.tsx @@ -3,12 +3,17 @@ import { Tab, Tabs } from 'fumadocs-ui/components/tabs'; import type { MDXComponents } from 'mdx/types'; import { PlatformOnly } from '@/components/platform'; +function DevOnly({ children }: { children: React.ReactNode }) { + return process.env.NEXT_PUBLIC_WORKSHOP_ENV === 'dev' ? children : null; +} + export function getMDXComponents(components?: MDXComponents) { return { ...defaultMdxComponents, Tab, Tabs, PlatformOnly, + DevOnly, ...components, } satisfies MDXComponents; } diff --git a/workshops/build_workshop/playbook/src/lib/layout.shared.tsx b/workshops/build_workshop/playbook/src/lib/layout.shared.tsx index f417340..6efeb6d 100644 --- a/workshops/build_workshop/playbook/src/lib/layout.shared.tsx +++ b/workshops/build_workshop/playbook/src/lib/layout.shared.tsx @@ -9,17 +9,16 @@ export function baseOptions(): BaseLayoutProps { }, links: [ { - text: 'Overview', + text: 'Use cases', url: '/docs', - active: 'nested-url', }, { - text: 'Learner track', - url: '/docs/learner', + text: 'AI SRE', + url: '/docs/ai-sre', }, { - text: 'Instructor track', - url: '/docs/instructor', + text: 'Polymarket', + url: '/docs/polymarket', }, ], githubUrl: `https://github.com/${gitConfig.user}/${gitConfig.repo}`, diff --git a/workshops/build_workshop/playbook/src/lib/shared.ts b/workshops/build_workshop/playbook/src/lib/shared.ts index 3290cce..b5fd243 100644 --- a/workshops/build_workshop/playbook/src/lib/shared.ts +++ b/workshops/build_workshop/playbook/src/lib/shared.ts @@ -9,7 +9,7 @@ export const docsContentRoute = '/llms.mdx/docs'; export const gitConfig = { user: 'ClickHouse', repo: 'ClickHouse_Demos', - branch: 'build-workshop-v1', + branch: process.env.NEXT_PUBLIC_WORKSHOP_BRANCH ?? 'build-workshop-v1', }; // The repo participants clone; the app lives at workshops/build_workshop/app. diff --git a/workshops/build_workshop/playbook/src/lib/source.ts b/workshops/build_workshop/playbook/src/lib/source.ts index a00a3fc..158a02d 100644 --- a/workshops/build_workshop/playbook/src/lib/source.ts +++ b/workshops/build_workshop/playbook/src/lib/source.ts @@ -10,6 +10,13 @@ export const source = loader({ plugins: [lucideIconsPlugin()], }); +export function isHiddenRehearsal(slugs: string[] | undefined): boolean { + return ( + process.env.NEXT_PUBLIC_WORKSHOP_ENV !== 'dev' && + slugs?.join('/') === 'polymarket/rehearsal' + ); +} + export function getPageImage(page: (typeof source)['$inferPage']) { const segments = [...page.slugs, 'image.png']; diff --git a/workshops/build_workshop/polymarket/.env.polymarket.example b/workshops/build_workshop/polymarket/.env.polymarket.example new file mode 100644 index 0000000..3505a9d --- /dev/null +++ b/workshops/build_workshop/polymarket/.env.polymarket.example @@ -0,0 +1,18 @@ +# ClickHouse Cloud. Use the bare host name, without https:// or a port. +CLICKHOUSE_HOST= +CLICKHOUSE_PORT=8443 +CLICKHOUSE_USER=default +CLICKHOUSE_PASSWORD= +CLICKHOUSE_DATABASE=polymarket +CLICKHOUSE_SECURE=true + +# Collector. Start with live; switch to fixture only when a classroom network +# blocks Polymarket or the watched markets are quiet. +POLYMARKET_MODE=live +POLYMARKET_MARKET_COUNT=5 +POLYMARKET_RECONCILE_SECONDS=10 +POLYMARKET_BOOK_FALLBACK_SECONDS=30 +POLYMARKET_STALL_SECONDS=30 +POLYMARKET_DEDUPE_MINUTES=15 +POLYMARKET_INITIAL_LOOKBACK_MINUTES=10 +POLYMARKET_HEALTH_PORT=8090 diff --git a/workshops/build_workshop/polymarket/.gitignore b/workshops/build_workshop/polymarket/.gitignore new file mode 100644 index 0000000..0b2e69e --- /dev/null +++ b/workshops/build_workshop/polymarket/.gitignore @@ -0,0 +1,3 @@ +.env.polymarket +.pytest_cache/ +__pycache__/ diff --git a/workshops/build_workshop/polymarket/README.md b/workshops/build_workshop/polymarket/README.md new file mode 100644 index 0000000..645eb15 --- /dev/null +++ b/workshops/build_workshop/polymarket/README.md @@ -0,0 +1,60 @@ +# Polymarket real-time analytics track + +This directory backs the dedicated Polymarket learner and instructor tracks published +at `/docs/polymarket`. It reads public market data only. It does not authenticate to +Polymarket, place orders, create wallets, or provide financial advice. + +## Data flow + +```text +Gamma discovery ───────────┐ +CLOB WebSocket + book REST ├─ collector ─ ClickHouse Cloud +Data API trade REST ───────┘ ├─ price_ticks -> midpoint MV + └─ trades -> saved queries/dashboard +``` + +The WebSocket provides low-latency quote updates. Data API polling reconciles public +trades with a five-second overlap. When the socket stalls, CLOB REST maintains BBO and +spread freshness. Deterministic event/trade IDs, a recent-ID hydrate on restart, and a +stable ClickHouse insert deduplication token keep retries safe before the incremental MV +sees rows. + +The public Data API does not expose a stable per-fill ID. `trade_id` therefore hashes +the documented transaction, token, wallet, side, price, size, and timestamp fields. +Rows indistinguishable across all of those public fields are treated as one observation. + +## Local runtime + +Only `collector` runs locally. ClickHouse is always Cloud-hosted. + +```bash +cp .env.polymarket.example .env.polymarket +# Fill the ClickHouse Cloud host and password. +set -a; source ./.env.polymarket; set +a +./preflight.sh +docker compose --env-file .env.polymarket up -d --build collector +curl --fail --silent http://localhost:8090/health | python3 -m json.tool +``` + +## Tests + +```bash +cd collector +python3 -m pip install -r requirements-dev.txt +python3 -m pytest -q +cd .. +./test-clickhouse.sh +``` + +`test-clickhouse.sh` starts a disposable ClickHouse 26.3 container, exercises the real +Python storage adapter, applies the schema, and executes canonical plus learner-only SQL. + +## Files + +- `collector/collector/`: source API, normalization, retry/deduplication, storage, and health code. +- `collector/tests/`: deterministic unit and fake-source integration tests. +- `db/schema.sql`: executable maintainer/test copy of the learner Module 02 SQL. +- `db/queries.sql`: executable maintainer/test copy of the learner Module 05/06 SQL. +- `docker-compose.yml`: the single stateless collector service. +- `.env.polymarket.example`: Cloud and collector configuration contract. +- `preflight.sh`: Docker, ClickHouse Cloud, and public-source reachability checks. diff --git a/workshops/build_workshop/polymarket/collector/Dockerfile b/workshops/build_workshop/polymarket/collector/Dockerfile new file mode 100644 index 0000000..58be907 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY collector ./collector + +RUN useradd --create-home --uid 10001 workshop +USER workshop + +EXPOSE 8090 + +CMD ["python", "-m", "collector"] diff --git a/workshops/build_workshop/polymarket/collector/collector/__init__.py b/workshops/build_workshop/polymarket/collector/collector/__init__.py new file mode 100644 index 0000000..4651b59 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/__init__.py @@ -0,0 +1 @@ +"""Polymarket market-data collector for the ClickHouse Cloud workshop.""" diff --git a/workshops/build_workshop/polymarket/collector/collector/__main__.py b/workshops/build_workshop/polymarket/collector/collector/__main__.py new file mode 100644 index 0000000..0e903f2 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/__main__.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import asyncio +import signal + +import aiohttp + +from .api import PolymarketAPI +from .config import Settings +from .health import HealthState, start_health_server +from .service import CollectorService, log +from .storage import ClickHouseStorage + + +async def main() -> None: + settings = Settings.from_env() + health = HealthState(queue_capacity=settings.queue_capacity) + runner = await start_health_server(health, settings.health_port) + storage = ClickHouseStorage(settings) + timeout = aiohttp.ClientTimeout(total=20, connect=10) + async with aiohttp.ClientSession(timeout=timeout) as session: + service = CollectorService( + settings, + PolymarketAPI(session), + storage, + health, + ) + loop = asyncio.get_running_loop() + for name in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(name, service.stop.set) + try: + await service.run() + finally: + await storage.close() + await runner.cleanup() + log("collector_stopped") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/workshops/build_workshop/polymarket/collector/collector/api.py b/workshops/build_workshop/polymarket/collector/collector/api.py new file mode 100644 index 0000000..3b648a4 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/api.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import asyncio +import math +from collections.abc import AsyncIterator +from typing import Any + +import aiohttp + +from .models import MarketToken, normalize_markets + + +GAMMA_MARKETS = "https://gamma-api.polymarket.com/markets" +DATA_TRADES = "https://data-api.polymarket.com/trades" +CLOB_BOOK = "https://clob.polymarket.com/book" +CLOB_WEBSOCKET = "wss://ws-subscriptions-clob.polymarket.com/ws/market" + + +class SourceError(RuntimeError): + pass + + +class RateLimited(SourceError): + def __init__(self, retry_after: float): + super().__init__(f"source rate limited; retry after {retry_after:g}s") + self.retry_after = retry_after + + +class PolymarketAPI: + def __init__(self, session: aiohttp.ClientSession): + self.session = session + + async def _get_json(self, url: str, params: dict[str, Any]) -> Any: + async with self.session.get(url, params=params) as response: + if response.status == 429: + raw = response.headers.get("Retry-After", "1") + try: + retry_after = float(raw) + except ValueError: + retry_after = 1.0 + if not math.isfinite(retry_after): + retry_after = 1.0 + retry_after = min(60.0, max(1.0, retry_after)) + raise RateLimited(retry_after) + if response.status >= 400: + body = (await response.text())[:200] + raise SourceError(f"{url} returned HTTP {response.status}: {body}") + try: + return await response.json() + except (aiohttp.ContentTypeError, ValueError) as exc: + body = (await response.text())[:200] + raise SourceError(f"{url} returned invalid JSON: {body}") from exc + + async def discover(self, market_count: int) -> list[MarketToken]: + payload = await self._get_json( + GAMMA_MARKETS, + { + "active": "true", + "closed": "false", + "limit": max(20, market_count), + "order": "volume24hr", + "ascending": "false", + }, + ) + if not isinstance(payload, list): + raise SourceError("Gamma /markets response must be an array") + return normalize_markets(payload, market_count) + + async def trades( + self, condition_id: str, start: int, end: int + ) -> list[dict[str, Any]]: + return await self._trades_window(condition_id, start, end, depth=0) + + async def _trades_window( + self, condition_id: str, start: int, end: int, depth: int + ) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + offset = 0 + limit = 10_000 + while offset <= 10_000: + page = await self._get_json( + DATA_TRADES, + { + "market": condition_id, + "start": max(0, start), + "end": end, + "limit": limit, + "offset": offset, + }, + ) + if not isinstance(page, list): + raise SourceError("Data API /trades response must be an array") + rows.extend(page) + if len(page) < limit: + break + if offset == 10_000: + if start >= end or depth >= 20: + raise SourceError( + "Data API /trades exceeds 11,000 rows inside one second; " + "the checkpoint was not advanced" + ) + midpoint = start + (end - start) // 2 + older = await self._trades_window( + condition_id, start, midpoint, depth + 1 + ) + newer = await self._trades_window( + condition_id, midpoint + 1, end, depth + 1 + ) + return [*older, *newer] + offset += limit + return rows + + async def book(self, token_id: int) -> dict[str, Any]: + payload = await self._get_json(CLOB_BOOK, {"token_id": str(token_id)}) + if not isinstance(payload, dict): + raise SourceError("CLOB /book response must be an object") + payload.setdefault("asset_id", str(token_id)) + return payload + + async def websocket_messages( + self, + token_ids: list[int], + stall_seconds: float, + heartbeat_seconds: float = 10, + ) -> AsyncIterator[dict[str, Any]]: + async with self.session.ws_connect( + CLOB_WEBSOCKET, + heartbeat=None, + timeout=aiohttp.ClientWSTimeout(ws_close=5), + ) as socket: + await socket.send_json( + { + "assets_ids": [str(token_id) for token_id in token_ids], + "type": "market", + "custom_feature_enabled": True, + } + ) + + async def heartbeat() -> None: + while not socket.closed: + await asyncio.sleep(heartbeat_seconds) + await socket.send_str("PING") + + heartbeat_task = asyncio.create_task(heartbeat()) + loop = asyncio.get_running_loop() + last_market_data = loop.time() + try: + while not socket.closed: + remaining = stall_seconds - (loop.time() - last_market_data) + if remaining <= 0: + raise SourceError( + f"WebSocket emitted no market data for {stall_seconds}s" + ) + try: + message = await asyncio.wait_for( + socket.receive(), timeout=remaining + ) + except TimeoutError as exc: + raise SourceError( + f"WebSocket emitted no market data for {stall_seconds}s" + ) from exc + if message.type == aiohttp.WSMsgType.TEXT: + if message.data == "PONG": + continue + try: + payload = message.json() + except ValueError as exc: + raise SourceError( + f"WebSocket returned invalid JSON: {message.data[:200]}" + ) from exc + items = payload if isinstance(payload, list) else [payload] + market_items = [item for item in items if isinstance(item, dict)] + if market_items: + last_market_data = loop.time() + for item in market_items: + yield item + elif message.type in { + aiohttp.WSMsgType.CLOSED, + aiohttp.WSMsgType.CLOSE, + aiohttp.WSMsgType.ERROR, + }: + raise SourceError("WebSocket closed before the collector stopped") + finally: + heartbeat_task.cancel() + await asyncio.gather(heartbeat_task, return_exceptions=True) diff --git a/workshops/build_workshop/polymarket/collector/collector/config.py b/workshops/build_workshop/polymarket/collector/collector/config.py new file mode 100644 index 0000000..6df80e3 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/config.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _positive_int(name: str, default: int) -> int: + raw = os.getenv(name, str(default)) + try: + value = int(raw) + except ValueError as exc: + raise ValueError(f"{name} must be an integer") from exc + if value <= 0: + raise ValueError(f"{name} must be greater than zero") + return value + + +def _required(name: str) -> str: + value = os.getenv(name, "").strip() + if not value: + raise ValueError(f"{name} is required") + return value + + +@dataclass(frozen=True) +class Settings: + clickhouse_host: str + clickhouse_port: int + clickhouse_user: str + clickhouse_password: str + clickhouse_database: str + clickhouse_secure: bool + mode: str + market_count: int + reconcile_seconds: int + book_fallback_seconds: int + stall_seconds: int + dedupe_minutes: int + initial_lookback_minutes: int + health_port: int + queue_capacity: int = 10_000 + + @classmethod + def from_env(cls) -> "Settings": + mode = os.getenv("POLYMARKET_MODE", "live").strip().lower() + if mode not in {"live", "fixture"}: + raise ValueError("POLYMARKET_MODE must be live or fixture") + secure = os.getenv("CLICKHOUSE_SECURE", "true").strip().lower() + if secure not in {"true", "false"}: + raise ValueError("CLICKHOUSE_SECURE must be true or false") + return cls( + clickhouse_host=_required("CLICKHOUSE_HOST"), + clickhouse_port=_positive_int("CLICKHOUSE_PORT", 8443), + clickhouse_user=os.getenv("CLICKHOUSE_USER", "default").strip() or "default", + clickhouse_password=_required("CLICKHOUSE_PASSWORD"), + clickhouse_database=os.getenv("CLICKHOUSE_DATABASE", "polymarket").strip() + or "polymarket", + clickhouse_secure=secure == "true", + mode=mode, + market_count=_positive_int("POLYMARKET_MARKET_COUNT", 5), + reconcile_seconds=_positive_int("POLYMARKET_RECONCILE_SECONDS", 10), + book_fallback_seconds=_positive_int( + "POLYMARKET_BOOK_FALLBACK_SECONDS", 30 + ), + stall_seconds=_positive_int("POLYMARKET_STALL_SECONDS", 30), + dedupe_minutes=_positive_int("POLYMARKET_DEDUPE_MINUTES", 15), + initial_lookback_minutes=_positive_int( + "POLYMARKET_INITIAL_LOOKBACK_MINUTES", 10 + ), + health_port=_positive_int("POLYMARKET_HEALTH_PORT", 8090), + ) diff --git a/workshops/build_workshop/polymarket/collector/collector/health.py b/workshops/build_workshop/polymarket/collector/collector/health.py new file mode 100644 index 0000000..6654109 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/health.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from typing import Any + +from aiohttp import web + + +def _iso(value: datetime | None) -> str | None: + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") if value else None + + +@dataclass +class HealthState: + status: str = "starting" + reason: str = "initializing" + websocket: str = "connecting" + last_websocket_event_at: datetime | None = None + last_trade_reconcile_at: datetime | None = None + last_book_fallback_at: datetime | None = None + last_clickhouse_write_at: datetime | None = None + queue_depth: int = 0 + queue_capacity: int = 10_000 + watched_markets: int = 0 + watched_tokens: int = 0 + fresh_tokens: int = 0 + websocket_fresh_tokens: int = 0 + source_parse_errors_total: int = 0 + source_errors_total: int = 0 + clickhouse_errors_total: int = 0 + started_at: datetime = field(default_factory=lambda: datetime.now(UTC)) + + def payload(self) -> dict[str, Any]: + data = asdict(self) + for name in ( + "started_at", + "last_websocket_event_at", + "last_trade_reconcile_at", + "last_book_fallback_at", + "last_clickhouse_write_at", + ): + data[name] = _iso(getattr(self, name)) + return data + + +async def start_health_server(state: HealthState, port: int) -> web.AppRunner: + async def health(_: web.Request) -> web.Response: + status = 503 if state.status == "unhealthy" else 200 + return web.json_response(state.payload(), status=status) + + app = web.Application() + app.router.add_get("/health", health) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "0.0.0.0", port) + await site.start() + return runner diff --git a/workshops/build_workshop/polymarket/collector/collector/models.py b/workshops/build_workshop/polymarket/collector/collector/models.py new file mode 100644 index 0000000..483d14b --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/models.py @@ -0,0 +1,398 @@ +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from decimal import ROUND_HALF_EVEN, Decimal, DecimalException +from typing import Any, Iterable + + +ZERO = Decimal("0") +ONE = Decimal("1") +MAX_SIZE = Decimal("1e20") +MAX_VOLUME_24H = Decimal("1e30") - Decimal("0.00000001") +MIDPOINT_QUANTUM = Decimal("0.000000000001") +EARLIEST_SOURCE_TIME = datetime(2020, 1, 1, tzinfo=UTC) +UINT256_MAX = 2**256 - 1 +MAX_RAW_PAYLOAD_CHARS = 16_384 +HEX_ID = re.compile(r"^0x[0-9a-fA-F]+$") + + +def fixed_hex(value: Any, digits: int, field: str) -> str: + text = str(value or "") + if len(text) != digits + 2 or not HEX_ID.fullmatch(text): + raise ValueError(f"{field} must be a 0x-prefixed {digits}-digit hex value") + return text + + +def uint256(value: Any, field: str = "token_id") -> int: + try: + number = int(value) + except (OverflowError, TypeError, ValueError) as exc: + raise ValueError(f"{field} must be an integer") from exc + if not 0 < number <= UINT256_MAX: + raise ValueError(f"{field} must be between 1 and 2^256 - 1") + return number + + +def uint64(value: Any, field: str) -> int: + try: + number = int(value) + except (OverflowError, TypeError, ValueError) as exc: + raise ValueError(f"{field} must be an integer") from exc + if not 0 <= number <= 2**64 - 1: + raise ValueError(f"{field} must be between 0 and 2^64 - 1") + return number + + +def decimal_text(value: Any, default: str = "0", max_scale: int | None = None) -> str: + if value in {None, ""}: + return default + raw = str(value) + if len(raw) > 128: + raise ValueError("decimal input is too long") + try: + normalized = Decimal(raw).normalize() + except (DecimalException, ValueError) as exc: + raise ValueError(f"invalid decimal value: {value!r}") from exc + if not normalized.is_finite(): + raise ValueError(f"decimal value must be finite: {value!r}") + if abs(normalized.adjusted()) > 30 or len(normalized.as_tuple().digits) > 38: + raise ValueError("decimal value exceeds the supported precision") + if max_scale is not None and -normalized.as_tuple().exponent > max_scale: + raise ValueError(f"decimal value exceeds scale {max_scale}") + return format(normalized, "f") + + +def parse_timestamp(value: Any) -> datetime: + if value in {None, ""}: + raise ValueError("source timestamp is required") + text = str(value) + try: + if text.isdigit(): + number = int(text) + if number > 10_000_000_000: + number /= 1000 + parsed = datetime.fromtimestamp(number, UTC) + else: + parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("source timestamp must include a timezone") + parsed = parsed.astimezone(UTC) + except (OverflowError, OSError, ValueError) as exc: + raise ValueError(f"invalid source timestamp: {value!r}") from exc + if parsed < EARLIEST_SOURCE_TIME or parsed > datetime.now(UTC) + timedelta(minutes=5): + raise ValueError("source timestamp is outside the accepted time window") + return parsed + + +def stable_id(*parts: Any) -> str: + canonical = "\x1f".join("" if part is None else str(part) for part in parts) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def stable_batch_token(ids: Iterable[str]) -> str: + return stable_id(*sorted(ids)) + + +def _json_list(value: Any, field: str) -> list[Any]: + if isinstance(value, list): + return value + if isinstance(value, str): + parsed = json.loads(value) + if isinstance(parsed, list): + return parsed + raise ValueError(f"{field} must be a JSON array") + + +@dataclass(frozen=True) +class MarketToken: + market_id: int + condition_id: str + token_id: int + outcome: str + question: str + slug: str + active: bool + accepting_orders: bool + volume_24h: str + + +def normalize_markets(payload: list[dict[str, Any]], limit: int) -> list[MarketToken]: + tokens: list[MarketToken] = [] + for market in payload: + try: + if not isinstance(market, dict): + continue + if ( + not market.get("active") + or market.get("closed") + or not market.get("acceptingOrders") + ): + continue + token_ids = _json_list(market.get("clobTokenIds"), "clobTokenIds") + outcomes = _json_list(market.get("outcomes"), "outcomes") + condition_id = fixed_hex(market.get("conditionId"), 64, "conditionId") + if not token_ids or len(token_ids) != len(outcomes): + continue + normalized_token_ids = [uint256(token_id) for token_id in token_ids] + market_id = uint64(market["id"], "market id") + volume_24h = Decimal( + decimal_text(market.get("volume24hr"), max_scale=8) + ) + if not ZERO <= volume_24h <= MAX_VOLUME_24H: + raise ValueError("volume24hr exceeds Decimal128(8)") + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + continue + for token_id, outcome in zip(normalized_token_ids, outcomes, strict=True): + tokens.append( + MarketToken( + market_id=market_id, + condition_id=condition_id, + token_id=token_id, + outcome=str(outcome), + question=str(market.get("question", "")), + slug=str(market.get("slug", "")), + active=True, + accepting_orders=bool(market.get("acceptingOrders", False)), + volume_24h=format(volume_24h, "f"), + ) + ) + if len({token.condition_id for token in tokens}) >= limit: + break + selected = {token.condition_id for token in tokens[: limit * 2]} + return [token for token in tokens if token.condition_id in selected][: limit * 2] + + +def build_tick( + *, + event_kind: str, + source: str, + condition_id: str, + token_id: Any, + timestamp: Any, + price: Any = None, + size: Any = None, + side: Any = None, + best_bid: Any = None, + best_ask: Any = None, + source_hash: Any = None, + raw: Any, +) -> dict[str, Any]: + condition_id = fixed_hex(condition_id, 64, "condition_id") + normalized_token_id = uint256(token_id) + event_at = parse_timestamp(timestamp) + price_text = decimal_text(price, max_scale=12) + size_text = decimal_text(size, max_scale=8) + bid_text = decimal_text(best_bid, max_scale=12) + ask_text = decimal_text(best_ask, max_scale=12) + bid = Decimal(bid_text) + ask = Decimal(ask_text) + normalized_price = Decimal(price_text) + normalized_size = Decimal(size_text) + if not ZERO <= normalized_price <= ONE: + raise ValueError("price must be between 0 and 1") + if not ZERO <= bid <= ONE or not ZERO <= ask <= ONE: + raise ValueError("best bid and ask must be between 0 and 1") + if bid > ZERO and ask > ZERO and bid > ask: + raise ValueError("best bid cannot exceed best ask") + if not ZERO <= normalized_size <= MAX_SIZE: + raise ValueError("size must be between 0 and 1e20") + midpoint = ( + decimal_text( + ((bid + ask) / 2).quantize(MIDPOINT_QUANTUM, rounding=ROUND_HALF_EVEN), + max_scale=12, + ) + if bid > ZERO and ask > ZERO + else "0" + ) + normalized_side = str(side or "UNKNOWN").upper() + if normalized_side not in {"BUY", "SELL"}: + normalized_side = "UNKNOWN" + event_id = stable_id( + condition_id, + normalized_token_id, + event_kind, + event_at.isoformat(), + price_text, + size_text, + normalized_side, + bid_text, + ask_text, + source_hash or "", + ) + return { + "event_id": event_id, + "condition_id": condition_id, + "token_id": normalized_token_id, + "event_at": event_at, + "observed_at": datetime.now(UTC), + "event_kind": event_kind, + "source": source, + "price": price_text, + "size": size_text, + "side": normalized_side, + "best_bid": bid_text, + "best_ask": ask_text, + "midpoint": midpoint, + "source_hash": str(source_hash or "")[:256], + "raw_payload": json.dumps(raw, separators=(",", ":"), sort_keys=True)[ + :MAX_RAW_PAYLOAD_CHARS + ], + } + + +def normalize_ws_message(message: dict[str, Any]) -> list[dict[str, Any]]: + event_type = message.get("event_type") or message.get("type") + payload = message.get("payload") if isinstance(message.get("payload"), dict) else message + condition_id = str(payload.get("market", "")) + timestamp = payload.get("timestamp") + if event_type == "book": + bids = payload.get("bids") or [] + asks = payload.get("asks") or [] + if not isinstance(bids, list) or not all(isinstance(row, dict) for row in bids): + raise ValueError("book bids must be an array of objects") + if not isinstance(asks, list) or not all(isinstance(row, dict) for row in asks): + raise ValueError("book asks must be an array of objects") + best_bid = max((Decimal(decimal_text(row.get("price"))) for row in bids), default=ZERO) + best_ask = min((Decimal(decimal_text(row.get("price"))) for row in asks), default=ZERO) + return [ + build_tick( + event_kind="book_snapshot", + source="WEBSOCKET", + condition_id=condition_id, + token_id=payload.get("asset_id") or payload.get("tokenId") or payload.get("token_id"), + timestamp=timestamp, + best_bid=best_bid, + best_ask=best_ask, + source_hash=payload.get("hash"), + raw=message, + ) + ] + if event_type == "price_change": + changes = payload.get("price_changes") or payload.get("priceChanges") or [] + if not isinstance(changes, list) or not all( + isinstance(change, dict) for change in changes + ): + raise ValueError("price changes must be an array of objects") + return [ + build_tick( + event_kind="price_change", + source="WEBSOCKET", + condition_id=condition_id, + token_id=change.get("asset_id") or change.get("tokenId"), + timestamp=timestamp, + price=change.get("price"), + size=change.get("size"), + side=change.get("side"), + best_bid=change.get("best_bid") or change.get("bestBid"), + best_ask=change.get("best_ask") or change.get("bestAsk"), + source_hash=change.get("hash"), + raw={ + "event_type": "price_change", + "market": condition_id, + "timestamp": timestamp, + "change": change, + }, + ) + for change in changes + ] + if event_type == "last_trade_price": + return [ + build_tick( + event_kind="last_trade_price", + source="WEBSOCKET", + condition_id=condition_id, + token_id=payload.get("asset_id") or payload.get("tokenId") or payload.get("token_id"), + timestamp=timestamp, + price=payload.get("price"), + size=payload.get("size"), + side=payload.get("side"), + source_hash=payload.get("transaction_hash") or payload.get("transactionHash"), + raw=message, + ) + ] + if event_type == "best_bid_ask": + return [ + build_tick( + event_kind="best_bid_ask", + source="WEBSOCKET", + condition_id=condition_id, + token_id=payload.get("asset_id") or payload.get("tokenId") or payload.get("token_id"), + timestamp=timestamp, + best_bid=payload.get("best_bid") or payload.get("bestBid"), + best_ask=payload.get("best_ask") or payload.get("bestAsk"), + raw=message, + ) + ] + return [] + + +def normalize_book(payload: dict[str, Any], condition_id: str) -> dict[str, Any]: + bids = payload.get("bids") or [] + asks = payload.get("asks") or [] + if not isinstance(bids, list) or not all(isinstance(row, dict) for row in bids): + raise ValueError("book bids must be an array of objects") + if not isinstance(asks, list) or not all(isinstance(row, dict) for row in asks): + raise ValueError("book asks must be an array of objects") + best_bid = max((Decimal(decimal_text(row.get("price"))) for row in bids), default=ZERO) + best_ask = min((Decimal(decimal_text(row.get("price"))) for row in asks), default=ZERO) + return build_tick( + event_kind="rest_book", + source="CLOB_REST", + condition_id=condition_id, + token_id=payload.get("asset_id") or payload.get("token_id"), + timestamp=payload.get("timestamp"), + best_bid=best_bid, + best_ask=best_ask, + source_hash=payload.get("hash"), + raw=payload, + ) + + +def normalize_trade(payload: dict[str, Any]) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ValueError("trade must be an object") + event_at = parse_timestamp(payload.get("timestamp")) + condition_id = fixed_hex(payload.get("conditionId"), 64, "conditionId") + token_id = uint256(payload.get("asset"), "asset") + proxy_wallet = fixed_hex(payload.get("proxyWallet"), 40, "proxyWallet") + transaction_hash = fixed_hex( + payload.get("transactionHash"), + 64, + "transactionHash", + ) + price = decimal_text(payload.get("price"), max_scale=12) + size = decimal_text(payload.get("size"), max_scale=8) + if not ZERO <= Decimal(price) <= ONE: + raise ValueError("trade price must be between 0 and 1") + if not ZERO < Decimal(size) <= MAX_SIZE: + raise ValueError("trade size must be greater than 0 and at most 1e20") + side = str(payload.get("side") or "UNKNOWN").upper() + if side not in {"BUY", "SELL"}: + side = "UNKNOWN" + trade_id = stable_id( + transaction_hash, + token_id, + proxy_wallet, + side, + price, + size, + int(event_at.timestamp()), + ) + return { + "trade_id": trade_id, + "condition_id": condition_id, + "token_id": token_id, + "event_at": event_at, + "observed_at": datetime.now(UTC), + "proxy_wallet": proxy_wallet, + "side": side, + "price": price, + "size": size, + "outcome": str(payload.get("outcome", "")), + "transaction_hash": transaction_hash, + "title": str(payload.get("title", "")), + } diff --git a/workshops/build_workshop/polymarket/collector/collector/service.py b/workshops/build_workshop/polymarket/collector/collector/service.py new file mode 100644 index 0000000..e5cb882 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/service.py @@ -0,0 +1,797 @@ +from __future__ import annotations + +import asyncio +import json +from collections import OrderedDict +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from typing import Any + +import aiohttp + +from .api import PolymarketAPI, RateLimited, SourceError +from .config import Settings +from .health import HealthState +from .models import ( + MarketToken, + build_tick, + normalize_book, + normalize_trade, + normalize_ws_message, + stable_batch_token, +) +from .storage import MARKET_COLUMNS, TICK_COLUMNS, TRADE_COLUMNS, ClickHouseStorage + +WRITE_BATCH_ROWS = 1_000 +WRITE_COALESCE_SECONDS = 0.05 +SOURCE_CONCURRENCY = 5 +MARKET_REFRESH_SECONDS = 300 +RECONCILE_DEADLINE_SECONDS = 30 + + +def log(event: str, **fields: Any) -> None: + print( + json.dumps( + {"timestamp": datetime.now(UTC).isoformat(), "event": event, **fields}, + default=str, + sort_keys=True, + ), + flush=True, + ) + + +class DedupeWindow: + def __init__(self, capacity: int = 50_000): + self.capacity = capacity + self.committed: OrderedDict[str, None] = OrderedDict() + self.pending: set[str] = set() + + def hydrate(self, ids: list[str] | set[str]) -> None: + for item_id in ids: + self._commit_one(item_id) + + def reserve(self, ids: list[str]) -> list[str]: + fresh = [] + reserved = set(self.pending) + for item_id in ids: + if item_id in self.committed or item_id in reserved: + continue + fresh.append(item_id) + reserved.add(item_id) + self.pending.update(fresh) + return fresh + + def commit(self, ids: list[str]) -> None: + for item_id in ids: + self.pending.discard(item_id) + self._commit_one(item_id) + + def release(self, ids: list[str]) -> None: + self.pending.difference_update(ids) + + def _commit_one(self, item_id: str) -> None: + self.committed[item_id] = None + self.committed.move_to_end(item_id) + while len(self.committed) > self.capacity: + self.committed.popitem(last=False) + + +@dataclass +class Batch: + table: str + rows: list[dict[str, Any]] + columns: list[str] + ids: list[str] + token: str + checkpoints: dict[str, datetime] = field(default_factory=dict) + + +class CollectorService: + def __init__( + self, + settings: Settings, + api: PolymarketAPI, + storage: ClickHouseStorage, + health: HealthState, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ): + self.settings = settings + self.api = api + self.storage = storage + self.health = health + self.sleep = sleep + self.queue: asyncio.Queue[Batch] = asyncio.Queue() + self._capacity_changed = asyncio.Condition() + self._source_slots = asyncio.Semaphore(SOURCE_CONCURRENCY) + self._writer_stop = asyncio.Event() + self._queued_rows = 0 + self._write_failure_started_at: datetime | None = None + self._last_tick_enqueued_at: datetime | None = None + self._token_price_at: dict[int, datetime] = {} + self._token_websocket_at: dict[int, datetime] = {} + self._quiet_notice_at: datetime | None = None + self.dedupe = DedupeWindow() + self.tokens: list[MarketToken] = [] + self.checkpoints: dict[str, datetime] = {} + self.stop = asyncio.Event() + self._ws_healthy = False + + async def prepare(self) -> None: + await self.storage.ping() + await self.storage.assert_schema() + self.dedupe.hydrate(await self.storage.recent_ids(self.settings.dedupe_minutes)) + if self.settings.mode == "fixture": + self.tokens = fixture_markets(self.settings.market_count) + else: + self.tokens = await self.api.discover(self.settings.market_count) + if not self.tokens: + self.health.status = "degraded" + self.health.reason = "no_active_markets" + log( + "no_active_markets", + action="Set POLYMARKET_MODE=fixture and restart the collector", + ) + return + self.health.watched_markets = len({token.condition_id for token in self.tokens}) + self.health.watched_tokens = len(self.tokens) + self.checkpoints = await self.storage.trade_checkpoints( + {token.condition_id for token in self.tokens} + ) + now = datetime.now(UTC) + market_rows = [ + { + "market_id": token.market_id, + "condition_id": token.condition_id, + "token_id": token.token_id, + "outcome": token.outcome, + "question": token.question, + "slug": token.slug, + "active": token.active, + "accepting_orders": token.accepting_orders, + "volume_24h": token.volume_24h, + "observed_at": now, + } + for token in self.tokens + ] + ids = [f"market:{row['condition_id']}:{row['token_id']}:{now.isoformat()}" for row in market_rows] + await self.storage.insert("markets", market_rows, MARKET_COLUMNS, stable_batch_token(ids)) + self.health.last_clickhouse_write_at = datetime.now(UTC) + self.health.status = "live" if self.settings.mode == "live" else "fixture" + if self.settings.mode == "fixture": + self.health.websocket = "fixture" + self.health.reason = "ready" + log("collector_ready", mode=self.settings.mode, markets=self.health.watched_markets) + + async def run(self) -> None: + await self.prepare() + writer = asyncio.create_task(self.writer_loop(), name="clickhouse-writer") + producers = [asyncio.create_task(self.health_loop(), name="health-monitor")] + if not self.tokens: + producers.append( + asyncio.create_task( + self.discovery_retry_loop(), name="market-discovery-retry" + ) + ) + elif self.settings.mode == "fixture": + producers.append(asyncio.create_task(self.fixture_loop(), name="fixture-feed")) + else: + producers.extend( + [ + asyncio.create_task(self.websocket_loop(), name="market-websocket"), + asyncio.create_task( + self.trade_reconcile_loop(), name="trade-reconciliation" + ), + asyncio.create_task(self.book_fallback_loop(), name="book-fallback"), + asyncio.create_task( + self.market_refresh_loop(), name="market-refresh" + ), + ] + ) + stop_waiter = asyncio.create_task(self.stop.wait(), name="shutdown-signal") + failure: BaseException | None = None + try: + done, _ = await asyncio.wait( + [stop_waiter, writer, *producers], + return_when=asyncio.FIRST_COMPLETED, + ) + for task in done: + if task is stop_waiter: + continue + exception = task.exception() + if exception is not None: + failure = exception + break + if not self.stop.is_set(): + failure = RuntimeError( + f"collector task {task.get_name()} stopped unexpectedly" + ) + break + finally: + self.stop.set() + for task in producers: + task.cancel() + await asyncio.gather(*producers, return_exceptions=True) + drained = False + if not writer.done(): + try: + await asyncio.wait_for(asyncio.shield(self.queue.join()), timeout=30) + drained = True + except asyncio.TimeoutError: + log( + "shutdown_drain_timeout", + queued_rows=self._queued_rows, + action="REST reconciliation will recover trades after restart", + ) + self._writer_stop.set() + if not writer.done(): + if drained: + writer.cancel() + else: + try: + await asyncio.wait_for(writer, timeout=20) + except asyncio.TimeoutError: + writer.cancel() + await asyncio.gather(writer, return_exceptions=True) + stop_waiter.cancel() + await asyncio.gather(stop_waiter, return_exceptions=True) + if failure is not None: + raise failure + + async def discovery_retry_loop(self) -> None: + while not self.stop.is_set(): + await self.sleep(60) + try: + self.tokens = await self.api.discover(self.settings.market_count) + except Exception as exc: # noqa: BLE001 - loop must survive source failure + self._source_error("gamma_discovery", exc) + continue + if self.tokens: + log("markets_discovered", action="restart collector to subscribe") + self.stop.set() + + async def market_refresh_loop(self) -> None: + current = {(token.condition_id, token.token_id) for token in self.tokens} + while not self.stop.is_set(): + await self.sleep(MARKET_REFRESH_SECONDS) + try: + refreshed = await self.api.discover(self.settings.market_count) + except ( + SourceError, + aiohttp.ClientError, + OSError, + asyncio.TimeoutError, + ) as exc: + self._source_error("gamma_refresh", exc) + continue + selected = {(token.condition_id, token.token_id) for token in refreshed} + if selected and selected != current: + log( + "watched_markets_changed", + action="restart collector with the refreshed market set", + ) + self.stop.set() + return + + async def enqueue( + self, + table: str, + rows: list[dict[str, Any]], + columns: list[str], + id_field: str, + checkpoints: dict[str, datetime] | None = None, + ) -> None: + if not rows: + self.checkpoints.update(checkpoints or {}) + return + ids = [str(row[id_field]) for row in rows] + fresh_ids = self.dedupe.reserve(ids) + if not fresh_ids: + self.checkpoints.update(checkpoints or {}) + return + allowed = set(fresh_ids) + included: set[str] = set() + fresh_rows = [] + for row in rows: + row_id = str(row[id_field]) + if row_id in allowed and row_id not in included: + fresh_rows.append(row) + included.add(row_id) + chunks = [ + fresh_rows[index : index + self.settings.queue_capacity] + for index in range(0, len(fresh_rows), self.settings.queue_capacity) + ] + queued_ids: set[str] = set() + try: + for index, chunk in enumerate(chunks): + chunk_ids = [str(row[id_field]) for row in chunk] + if not await self._reserve_capacity(len(chunk)): + break + batch = Batch( + table=table, + rows=chunk, + columns=columns, + ids=chunk_ids, + token=stable_batch_token(chunk_ids), + checkpoints=(checkpoints or {}) if index == len(chunks) - 1 else {}, + ) + await self.queue.put(batch) + queued_ids.update(chunk_ids) + if table == "price_ticks": + self._last_tick_enqueued_at = datetime.now(UTC) + finally: + self.dedupe.release([item_id for item_id in fresh_ids if item_id not in queued_ids]) + + async def _reserve_capacity(self, rows: int) -> bool: + async with self._capacity_changed: + while ( + self._queued_rows + rows > self.settings.queue_capacity + and not self.stop.is_set() + ): + self.health.status = "unhealthy" + self.health.reason = "queue_full" + await self._capacity_changed.wait() + if self.stop.is_set(): + return False + self._queued_rows += rows + self.health.queue_depth = self._queued_rows + if self._queued_rows >= min(8_000, self.settings.queue_capacity): + self.health.status = "unhealthy" + self.health.reason = "queue_near_capacity" + return True + + async def _release_capacity(self, rows: int) -> None: + async with self._capacity_changed: + self._queued_rows = max(0, self._queued_rows - rows) + self.health.queue_depth = self._queued_rows + self._capacity_changed.notify_all() + + async def writer_loop(self) -> None: + deferred: Batch | None = None + while not self._writer_stop.is_set() or not self.queue.empty() or deferred: + first = deferred if deferred is not None else await self.queue.get() + deferred = None + batches = [first] + rows = len(first.rows) + await self.sleep(WRITE_COALESCE_SECONDS) + while rows < WRITE_BATCH_ROWS: + try: + candidate = self.queue.get_nowait() + except asyncio.QueueEmpty: + break + if ( + candidate.table != first.table + or candidate.columns != first.columns + or rows + len(candidate.rows) > WRITE_BATCH_ROWS + ): + deferred = candidate + break + batches.append(candidate) + rows += len(candidate.rows) + batch = self._combine_batches(batches) + delay = 1 + while not self._writer_stop.is_set(): + try: + await self.storage.insert( + batch.table, batch.rows, batch.columns, batch.token + ) + except Exception as exc: # noqa: BLE001 - retries preserve batch + self.health.clickhouse_errors_total += 1 + now = datetime.now(UTC) + self._write_failure_started_at = self._write_failure_started_at or now + failed_for = (now - self._write_failure_started_at).total_seconds() + self.health.status = "unhealthy" if failed_for >= 60 else "degraded" + self.health.reason = ( + "clickhouse_write_stalled" + if failed_for >= 60 + else "clickhouse_write_retrying" + ) + log( + "clickhouse_write_failed", + error=str(exc), + table=batch.table, + retry_seconds=delay, + ) + await self.sleep(delay) + delay = min(30, delay * 2) + continue + self.dedupe.commit(batch.ids) + self.checkpoints.update(batch.checkpoints) + self.health.last_clickhouse_write_at = datetime.now(UTC) + self._write_failure_started_at = None + await self._release_capacity(len(batch.rows)) + self._refresh_health(datetime.now(UTC)) + break + for _ in batches: + self.queue.task_done() + + @staticmethod + def _combine_batches(batches: list[Batch]) -> Batch: + if len(batches) == 1: + return batches[0] + rows = [row for batch in batches for row in batch.rows] + ids = [item_id for batch in batches for item_id in batch.ids] + checkpoints: dict[str, datetime] = {} + for batch in batches: + checkpoints.update(batch.checkpoints) + return Batch( + table=batches[0].table, + rows=rows, + columns=batches[0].columns, + ids=ids, + token=stable_batch_token(ids), + checkpoints=checkpoints, + ) + + async def health_loop(self) -> None: + while not self.stop.is_set(): + await self.sleep(5) + now = datetime.now(UTC) + self._refresh_health(now) + if self.settings.mode != "live": + continue + + startup_age = (now - self.health.started_at).total_seconds() + tick_age = ( + (now - self._last_tick_enqueued_at).total_seconds() + if self._last_tick_enqueued_at + else startup_age + ) + notice_age = ( + (now - self._quiet_notice_at).total_seconds() + if self._quiet_notice_at + else 61 + ) + if tick_age >= 60 and notice_age >= 60: + self._quiet_notice_at = now + log( + "quiet_feed", + action=( + "Set POLYMARKET_MODE=fixture in .env.polymarket and run " + "docker compose --env-file .env.polymarket up -d --force-recreate collector" + ), + ) + + def _refresh_health(self, now: datetime) -> None: + if self._write_failure_started_at is not None: + failed_for = (now - self._write_failure_started_at).total_seconds() + self.health.status = "unhealthy" if failed_for >= 60 else "degraded" + self.health.reason = ( + "clickhouse_write_stalled" + if failed_for >= 60 + else "clickhouse_write_retrying" + ) + return + if self._queued_rows >= min(8_000, self.settings.queue_capacity): + self.health.status = "unhealthy" + self.health.reason = "queue_near_capacity" + return + if self.settings.mode == "fixture": + self.health.status = "fixture" + self.health.reason = "ready" + return + if not self.tokens: + self.health.status = "degraded" + self.health.reason = "no_active_markets" + return + + startup_age = (now - self.health.started_at).total_seconds() + trade_age = ( + (now - self.health.last_trade_reconcile_at).total_seconds() + if self.health.last_trade_reconcile_at + else startup_age + ) + price_limit = max(60, self.settings.book_fallback_seconds * 3) + websocket_limit = max(60, self.settings.stall_seconds * 2) + self.health.fresh_tokens = sum( + (now - observed_at).total_seconds() <= price_limit + for observed_at in self._token_price_at.values() + ) + self.health.websocket_fresh_tokens = sum( + (now - observed_at).total_seconds() <= websocket_limit + for observed_at in self._token_websocket_at.values() + ) + price_fresh = ( + self.health.watched_tokens > 0 + and self.health.fresh_tokens == self.health.watched_tokens + ) + websocket_fresh = ( + self._ws_healthy + and self.health.websocket_fresh_tokens == self.health.watched_tokens + ) + trades_fresh = trade_age <= max(60, self.settings.reconcile_seconds * 3) + if not price_fresh and not trades_fresh: + self.health.status = "unhealthy" + self.health.reason = "all_sources_stale" + elif not price_fresh: + self.health.status = "degraded" + self.health.reason = "price_sources_stale" + elif not trades_fresh: + self.health.status = "degraded" + self.health.reason = "trade_reconciliation_stale" + elif not websocket_fresh: + self.health.status = "degraded" + self.health.reason = "websocket_stale_rest_active" + else: + self.health.status = "live" + self.health.reason = "sources_fresh" + + async def websocket_loop(self) -> None: + delay = 1 + token_ids = [token.token_id for token in self.tokens] + while not self.stop.is_set(): + self.health.websocket = "connecting" + try: + async for message in self.api.websocket_messages( + token_ids, self.settings.stall_seconds + ): + self._ws_healthy = True + self.health.websocket = "connected" + self.health.last_websocket_event_at = datetime.now(UTC) + self._refresh_health(datetime.now(UTC)) + try: + rows = normalize_ws_message(message) + except (KeyError, TypeError, ValueError) as exc: + self._parse_error("websocket", message, exc) + continue + rows = self._filter_watched(rows, "websocket") + await self.enqueue("price_ticks", rows, TICK_COLUMNS, "event_id") + delay = 1 + except ( + SourceError, + aiohttp.ClientError, + OSError, + asyncio.TimeoutError, + ) as exc: + self._ws_healthy = False + self.health.websocket = "retrying" + self._refresh_health(datetime.now(UTC)) + self._source_error("websocket", exc) + await self.sleep(delay) + delay = min(30, delay * 2) + + async def trade_reconcile_loop(self) -> None: + conditions = sorted({token.condition_id for token in self.tokens}) + while not self.stop.is_set(): + now = datetime.now(UTC) + results = await asyncio.gather( + *(self._reconcile_condition(condition_id, now) for condition_id in conditions) + ) + if results and all(results): + self.health.last_trade_reconcile_at = datetime.now(UTC) + self._refresh_health(datetime.now(UTC)) + await self.sleep(self.settings.reconcile_seconds) + + async def _reconcile_condition( + self, condition_id: str, now: datetime + ) -> bool: + checkpoint = self.checkpoints.get( + condition_id, + now - timedelta(minutes=self.settings.initial_lookback_minutes), + ) + start = int(checkpoint.timestamp()) - 5 + try: + async with self._source_slots: + async with asyncio.timeout( + max(RECONCILE_DEADLINE_SECONDS, self.settings.reconcile_seconds) + ): + payloads = await self.api.trades( + condition_id, start, int(now.timestamp()) + ) + except RateLimited as exc: + self._source_error("data_api", exc) + await self.sleep(exc.retry_after) + return False + except ( + KeyError, + TypeError, + ValueError, + SourceError, + aiohttp.ClientError, + OSError, + asyncio.TimeoutError, + ) as exc: + self._source_error("data_api", exc) + return False + + rows = [] + for payload in payloads: + try: + row = normalize_trade(payload) + except (KeyError, TypeError, ValueError) as exc: + self._parse_error("data_api", payload, exc) + continue + rows.extend(self._filter_watched([row], "data_api")) + if payloads and not rows: + return False + rows.sort(key=lambda row: row["event_at"]) + next_checkpoint = max((row["event_at"] for row in rows), default=now) + await self.enqueue( + "trades", + rows, + TRADE_COLUMNS, + "trade_id", + {condition_id: next_checkpoint}, + ) + return True + + async def book_fallback_loop(self) -> None: + token_conditions = {token.token_id: token.condition_id for token in self.tokens} + while not self.stop.is_set(): + await self.sleep(self.settings.book_fallback_seconds) + now = datetime.now(UTC) + stale_tokens = { + token_id: condition_id + for token_id, condition_id in token_conditions.items() + if token_id not in self._token_price_at + or (now - self._token_price_at[token_id]).total_seconds() + >= self.settings.book_fallback_seconds + } + if not stale_tokens: + continue + fetched = await asyncio.gather( + *( + self._fetch_book(token_id, condition_id) + for token_id, condition_id in stale_tokens.items() + ) + ) + rows = [row for row in fetched if row is not None] + await self.enqueue("price_ticks", rows, TICK_COLUMNS, "event_id") + if rows: + self.health.last_book_fallback_at = datetime.now(UTC) + self._refresh_health(datetime.now(UTC)) + + async def _fetch_book( + self, token_id: int, condition_id: str + ) -> dict[str, Any] | None: + try: + async with self._source_slots: + async with asyncio.timeout(max(10, self.settings.book_fallback_seconds)): + payload = await self.api.book(token_id) + row = normalize_book(payload, condition_id) + accepted = self._filter_watched([row], "clob_book") + if accepted and Decimal(str(accepted[0]["midpoint"])) > 0: + return accepted[0] + if accepted: + self._parse_error( + "clob_book", + accepted[0], + ValueError("book has no usable bid/ask midpoint"), + ) + return None + except RateLimited as exc: + self._source_error("clob_book", exc) + await self.sleep(exc.retry_after) + except ( + KeyError, + TypeError, + ValueError, + SourceError, + aiohttp.ClientError, + OSError, + asyncio.TimeoutError, + ) as exc: + self._source_error("clob_book", exc) + return None + + async def fixture_loop(self) -> None: + sequence = 0 + while not self.stop.is_set(): + now = datetime.now(UTC) + rows = fixture_ticks(self.tokens, sequence, now) + trades = fixture_trades(self.tokens, sequence, now) + await self.enqueue("price_ticks", rows, TICK_COLUMNS, "event_id") + await self.enqueue("trades", trades, TRADE_COLUMNS, "trade_id") + self.health.last_websocket_event_at = now + self.health.last_trade_reconcile_at = now + self.health.fresh_tokens = len(self.tokens) + sequence += 1 + await self.sleep(5) + + def _source_error(self, source: str, exc: Exception) -> None: + self.health.source_errors_total += 1 + log("source_error", source=source, error=str(exc)[:200]) + + def _filter_watched( + self, rows: list[dict[str, Any]], source: str + ) -> list[dict[str, Any]]: + token_conditions = { + token.token_id: token.condition_id for token in self.tokens + } + accepted = [] + for row in rows: + condition_id = token_conditions.get(row.get("token_id")) + if condition_id == row.get("condition_id"): + accepted.append(row) + continue + self._parse_error( + source, + row, + ValueError("event identifiers are outside the watched market set"), + ) + if source in {"websocket", "clob_book"}: + observed_at = datetime.now(UTC) + for row in accepted: + if Decimal(str(row.get("midpoint", "0"))) <= 0: + continue + token_id = row["token_id"] + self._token_price_at[token_id] = observed_at + if source == "websocket": + self._token_websocket_at[token_id] = observed_at + return accepted + + def _parse_error(self, source: str, payload: Any, exc: Exception) -> None: + self.health.source_parse_errors_total += 1 + preview = json.dumps(payload, default=str)[:200] + log("source_parse_error", source=source, error=str(exc), preview=preview) + + +def fixture_markets(count: int) -> list[MarketToken]: + rows = [] + for index in range(count): + condition = "0x" + f"{index + 1:064x}" + for outcome_index, outcome in enumerate(("Yes", "No")): + rows.append( + MarketToken( + market_id=9_000_000 + index, + condition_id=condition, + token_id=10_000_000 + index * 2 + outcome_index, + outcome=outcome, + question=f"Fixture market {index + 1}: will the signal move?", + slug=f"fixture-market-{index + 1}", + active=True, + accepting_orders=False, + volume_24h="1000", + ) + ) + return rows + + +def fixture_ticks( + tokens: list[MarketToken], sequence: int, now: datetime +) -> list[dict[str, Any]]: + rows = [] + for token in tokens: + base = 40 + ((sequence + token.market_id) % 20) + bid = f"0.{base:02d}" + ask = f"0.{base + 2:02d}" + rows.append( + build_tick( + event_kind="best_bid_ask", + source="FIXTURE", + condition_id=token.condition_id, + token_id=token.token_id, + timestamp=now.isoformat(), + best_bid=bid, + best_ask=ask, + source_hash=f"fixture-{sequence}-{token.token_id}", + raw={"fixture": True, "sequence": sequence}, + ) + ) + return rows + + +def fixture_trades( + tokens: list[MarketToken], sequence: int, now: datetime +) -> list[dict[str, Any]]: + rows = [] + for token in tokens[::2]: + price = 0.4 + ((sequence + token.market_id) % 20) / 100 + rows.append( + normalize_trade( + { + "transactionHash": "0x" + f"{sequence * 100 + token.market_id:064x}"[-64:], + "asset": str(token.token_id), + "proxyWallet": "0x" + "1" * 40, + "side": "BUY" if sequence % 2 == 0 else "SELL", + "price": str(price), + "size": str(10 + sequence % 5), + "timestamp": int(now.timestamp()), + "conditionId": token.condition_id, + "outcome": token.outcome, + "title": token.question, + } + ) + ) + return rows diff --git a/workshops/build_workshop/polymarket/collector/collector/storage.py b/workshops/build_workshop/polymarket/collector/collector/storage.py new file mode 100644 index 0000000..5fadcdd --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/collector/storage.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Iterable +from datetime import UTC, datetime, timedelta +from typing import Any + +import clickhouse_connect + +from .config import Settings + + +MARKET_COLUMNS = [ + "market_id", + "condition_id", + "token_id", + "outcome", + "question", + "slug", + "active", + "accepting_orders", + "volume_24h", + "observed_at", +] +TICK_COLUMNS = [ + "event_id", + "condition_id", + "token_id", + "event_at", + "observed_at", + "event_kind", + "source", + "price", + "size", + "side", + "best_bid", + "best_ask", + "midpoint", + "source_hash", + "raw_payload", +] +TRADE_COLUMNS = [ + "trade_id", + "condition_id", + "token_id", + "event_at", + "observed_at", + "proxy_wallet", + "side", + "price", + "size", + "outcome", + "transaction_hash", + "title", +] + + +def _text(value: Any) -> str: + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +class ClickHouseStorage: + def __init__(self, settings: Settings): + self.settings = settings + self.client = clickhouse_connect.get_client( + host=settings.clickhouse_host, + port=settings.clickhouse_port, + username=settings.clickhouse_user, + password=settings.clickhouse_password, + database=settings.clickhouse_database, + secure=settings.clickhouse_secure, + connect_timeout=10, + send_receive_timeout=15, + ) + + async def ping(self) -> None: + await asyncio.to_thread(self.client.command, "SELECT 1") + + async def assert_schema(self) -> None: + expected = { + "markets", + "price_ticks", + "trades", + "trades_clean", + "market_midpoints_1m", + "market_midpoints_1m_mv", + } + + def fetch() -> set[str]: + result = self.client.query( + "SELECT name FROM system.tables WHERE database = {db:String}", + parameters={"db": self.settings.clickhouse_database}, + ) + return {str(row[0]) for row in result.result_rows} + + actual = await asyncio.to_thread(fetch) + missing = sorted(expected - actual) + if missing: + raise RuntimeError( + "missing ClickHouse tables: " + ", ".join(missing) + "; run Module 02 SQL" + ) + + async def insert( + self, table: str, rows: list[dict[str, Any]], columns: list[str], token: str + ) -> None: + if not rows: + return + data = [[row[column] for column in columns] for row in rows] + await asyncio.to_thread( + self.client.insert, + f"{self.settings.clickhouse_database}.{table}", + data, + column_names=columns, + settings={ + "async_insert": 1, + "async_insert_deduplicate": 1, + "wait_for_async_insert": 1, + "insert_deduplication_token": token, + }, + ) + + async def recent_ids(self, minutes: int) -> list[str]: + since = datetime.now(UTC) - timedelta(minutes=minutes) + + def fetch() -> list[str]: + rows = self.client.query( + """ + SELECT id + FROM + ( + SELECT event_id AS id, observed_at + FROM price_ticks + WHERE observed_at >= {since:DateTime64(3)} + UNION ALL + SELECT trade_id AS id, observed_at + FROM trades FINAL + WHERE observed_at >= {since:DateTime64(3)} + ) + ORDER BY observed_at DESC + LIMIT 50000 + """, + parameters={"since": since}, + ).result_rows + return [_text(row[0]) for row in reversed(rows)] + + return await asyncio.to_thread(fetch) + + async def trade_checkpoints( + self, condition_ids: Iterable[str] + ) -> dict[str, datetime]: + ids = list(condition_ids) + if not ids: + return {} + + def fetch() -> dict[str, datetime]: + result = self.client.query( + """ + SELECT condition_id, max(event_at) + FROM trades FINAL + WHERE condition_id IN {ids:Array(String)} + GROUP BY condition_id + """, + parameters={"ids": ids}, + ) + return { + _text(row[0]): row[1].replace(tzinfo=UTC) + for row in result.result_rows + } + + return await asyncio.to_thread(fetch) + + async def close(self) -> None: + await asyncio.to_thread(self.client.close) diff --git a/workshops/build_workshop/polymarket/collector/requirements-dev.txt b/workshops/build_workshop/polymarket/collector/requirements-dev.txt new file mode 100644 index 0000000..263e48c --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest==9.0.2 +pytest-asyncio==1.4.0 diff --git a/workshops/build_workshop/polymarket/collector/requirements.txt b/workshops/build_workshop/polymarket/collector/requirements.txt new file mode 100644 index 0000000..552148d --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/requirements.txt @@ -0,0 +1,2 @@ +aiohttp==3.14.3 +clickhouse-connect==1.6.0 diff --git a/workshops/build_workshop/polymarket/collector/tests/test_api.py b/workshops/build_workshop/polymarket/collector/tests/test_api.py new file mode 100644 index 0000000..86dcf32 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/tests/test_api.py @@ -0,0 +1,226 @@ +import asyncio +from contextlib import asynccontextmanager + +import aiohttp +import pytest +from aiohttp import web + +import collector.api as api_module +from collector.api import PolymarketAPI, RateLimited, SourceError + + +@asynccontextmanager +async def server(routes): + app = web.Application() + for method, path, handler in routes: + app.router.add_route(method, path, handler) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", 0) + await site.start() + sockets = site._server.sockets # noqa: SLF001 - aiohttp exposes no public bound port + port = sockets[0].getsockname()[1] + try: + yield f"http://127.0.0.1:{port}" + finally: + await runner.cleanup() + + +@pytest.mark.asyncio +async def test_discover_sends_active_volume_query(monkeypatch): + async def markets(request): + assert request.query["active"] == "true" + assert request.query["closed"] == "false" + assert request.query["order"] == "volume24hr" + return web.json_response( + [ + { + "id": "1", + "conditionId": "0x" + "1" * 64, + "clobTokenIds": '["10", "11"]', + "outcomes": '["Yes", "No"]', + "active": True, + "closed": False, + "acceptingOrders": True, + } + ] + ) + + async with server([("GET", "/markets", markets)]) as base: + monkeypatch.setattr(api_module, "GAMMA_MARKETS", f"{base}/markets") + async with aiohttp.ClientSession() as session: + result = await PolymarketAPI(session).discover(1) + + assert [row.token_id for row in result] == [10, 11] + + +@pytest.mark.asyncio +async def test_trades_pages_until_short_page(monkeypatch): + calls = [] + + async def trades(request): + calls.append(int(request.query["offset"])) + if request.query["offset"] == "0": + return web.json_response([{"id": index} for index in range(10_000)]) + return web.json_response([{"id": 10_000}]) + + async with server([("GET", "/trades", trades)]) as base: + monkeypatch.setattr(api_module, "DATA_TRADES", f"{base}/trades") + async with aiohttp.ClientSession() as session: + result = await PolymarketAPI(session).trades("0x" + "1" * 64, 10, 20) + + assert len(result) == 10_001 + assert calls == [0, 10_000] + + +@pytest.mark.asyncio +async def test_trades_bisects_a_window_that_reaches_the_offset_cap(monkeypatch): + calls = [] + + async with aiohttp.ClientSession() as session: + api = PolymarketAPI(session) + + async def get_json(_url, params): + window = (params["start"], params["end"]) + calls.append((window, params["offset"])) + if window == (10, 20): + return [{"full": True}] * 10_000 + return [{"window": window}] + + monkeypatch.setattr(api, "_get_json", get_json) + rows = await api.trades("0x" + "1" * 64, 10, 20) + + assert [row["window"] for row in rows] == [(10, 15), (16, 20)] + assert ((10, 20), 10_000) in calls + + +@pytest.mark.asyncio +async def test_rate_limit_uses_retry_after(monkeypatch): + async def limited(_request): + return web.Response(status=429, headers={"Retry-After": "7"}) + + async with server([("GET", "/book", limited)]) as base: + monkeypatch.setattr(api_module, "CLOB_BOOK", f"{base}/book") + async with aiohttp.ClientSession() as session: + with pytest.raises(RateLimited) as error: + await PolymarketAPI(session).book(10) + + assert error.value.retry_after == 7 + + +@pytest.mark.asyncio +async def test_rate_limit_clamps_non_finite_retry_after(monkeypatch): + async def limited(_request): + return web.Response(status=429, headers={"Retry-After": "inf"}) + + async with server([("GET", "/book", limited)]) as base: + monkeypatch.setattr(api_module, "CLOB_BOOK", f"{base}/book") + async with aiohttp.ClientSession() as session: + with pytest.raises(RateLimited) as error: + await PolymarketAPI(session).book(10) + + assert error.value.retry_after == 1 + + +@pytest.mark.asyncio +async def test_invalid_json_is_source_error(monkeypatch): + async def invalid(_request): + return web.Response(text="not-json", content_type="application/json") + + async with server([("GET", "/book", invalid)]) as base: + monkeypatch.setattr(api_module, "CLOB_BOOK", f"{base}/book") + async with aiohttp.ClientSession() as session: + with pytest.raises(SourceError, match="invalid JSON"): + await PolymarketAPI(session).book(10) + + +@pytest.mark.asyncio +async def test_websocket_subscribes_heartbeats_and_yields_market_data(monkeypatch): + subscription = None + + async def websocket(request): + nonlocal subscription + socket = web.WebSocketResponse() + await socket.prepare(request) + subscription = await socket.receive_json() + ping = await socket.receive_str() + assert ping == "PING" + await socket.send_str("PONG") + await socket.send_json( + { + "event_type": "best_bid_ask", + "market": "0x" + "1" * 64, + "asset_id": "10", + "timestamp": "1782753357257", + "best_bid": "0.4", + "best_ask": "0.6", + } + ) + await socket.close() + return socket + + async with server([("GET", "/ws", websocket)]) as base: + monkeypatch.setattr( + api_module, "CLOB_WEBSOCKET", base.replace("http://", "ws://") + "/ws" + ) + async with aiohttp.ClientSession() as session: + messages = PolymarketAPI(session).websocket_messages( + [10, 11], stall_seconds=1, heartbeat_seconds=0.01 + ) + message = await anext(messages) + await messages.aclose() + + assert subscription == { + "assets_ids": ["10", "11"], + "type": "market", + "custom_feature_enabled": True, + } + assert message["event_type"] == "best_bid_ask" + + +@pytest.mark.asyncio +async def test_websocket_silent_stall_is_reported(monkeypatch): + async def websocket(request): + socket = web.WebSocketResponse() + await socket.prepare(request) + await socket.receive_json() + await asyncio.sleep(0.2) + await socket.close() + return socket + + async with server([("GET", "/ws", websocket)]) as base: + monkeypatch.setattr( + api_module, "CLOB_WEBSOCKET", base.replace("http://", "ws://") + "/ws" + ) + async with aiohttp.ClientSession() as session: + messages = PolymarketAPI(session).websocket_messages( + [10], stall_seconds=0.05 + ) + with pytest.raises(SourceError, match="no market data"): + await anext(messages) + + +@pytest.mark.asyncio +async def test_websocket_pongs_do_not_mask_a_market_data_stall(monkeypatch): + async def websocket(request): + socket = web.WebSocketResponse() + await socket.prepare(request) + await socket.receive_json() + while not socket.closed: + message = await socket.receive() + if message.type != aiohttp.WSMsgType.TEXT: + break + if message.data == "PING": + await socket.send_str("PONG") + return socket + + async with server([("GET", "/ws", websocket)]) as base: + monkeypatch.setattr( + api_module, "CLOB_WEBSOCKET", base.replace("http://", "ws://") + "/ws" + ) + async with aiohttp.ClientSession() as session: + messages = PolymarketAPI(session).websocket_messages( + [10], stall_seconds=0.05, heartbeat_seconds=0.01 + ) + with pytest.raises(SourceError, match="no market data"): + await anext(messages) diff --git a/workshops/build_workshop/polymarket/collector/tests/test_config.py b/workshops/build_workshop/polymarket/collector/tests/test_config.py new file mode 100644 index 0000000..3a0e63d --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/tests/test_config.py @@ -0,0 +1,40 @@ +import pytest + +from collector.config import Settings + + +def test_settings_load_cloud_defaults(monkeypatch): + monkeypatch.setenv("CLICKHOUSE_HOST", "abc.aws.clickhouse.cloud") + monkeypatch.setenv("CLICKHOUSE_PASSWORD", "secret") + + settings = Settings.from_env() + + assert settings.clickhouse_port == 8443 + assert settings.clickhouse_secure is True + assert settings.mode == "live" + assert settings.queue_capacity == 10_000 + + +@pytest.mark.parametrize( + ("name", "value", "message"), + [ + ("POLYMARKET_MODE", "paper", "must be live or fixture"), + ("CLICKHOUSE_SECURE", "maybe", "must be true or false"), + ("POLYMARKET_MARKET_COUNT", "0", "must be greater than zero"), + ], +) +def test_settings_reject_invalid_values(monkeypatch, name, value, message): + monkeypatch.setenv("CLICKHOUSE_HOST", "abc.aws.clickhouse.cloud") + monkeypatch.setenv("CLICKHOUSE_PASSWORD", "secret") + monkeypatch.setenv(name, value) + + with pytest.raises(ValueError, match=message): + Settings.from_env() + + +def test_settings_require_cloud_credentials(monkeypatch): + monkeypatch.delenv("CLICKHOUSE_HOST", raising=False) + monkeypatch.setenv("CLICKHOUSE_PASSWORD", "secret") + + with pytest.raises(ValueError, match="CLICKHOUSE_HOST is required"): + Settings.from_env() diff --git a/workshops/build_workshop/polymarket/collector/tests/test_models.py b/workshops/build_workshop/polymarket/collector/tests/test_models.py new file mode 100644 index 0000000..e64dcb4 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/tests/test_models.py @@ -0,0 +1,267 @@ +import json +from datetime import UTC, datetime + +import pytest + +from collector.models import ( + decimal_text, + normalize_book, + normalize_markets, + normalize_trade, + normalize_ws_message, + stable_batch_token, +) + + +def test_normalize_markets_parses_json_encoded_arrays_and_limit(): + payload = [ + { + "id": "42", + "conditionId": "0x" + "a" * 64, + "clobTokenIds": '["123", "456"]', + "outcomes": '["Yes", "No"]', + "question": "Will it ship?", + "slug": "will-it-ship", + "active": True, + "closed": False, + "acceptingOrders": True, + "volume24hr": 12.5, + }, + { + "id": "43", + "conditionId": "0x" + "b" * 64, + "clobTokenIds": '["789", "987"]', + "outcomes": '["Up", "Down"]', + "active": True, + "closed": False, + "acceptingOrders": True, + }, + ] + + result = normalize_markets(payload, 1) + + assert [(row.token_id, row.outcome) for row in result] == [(123, "Yes"), (456, "No")] + assert result[0].volume_24h == "12.5" + + +def test_normalize_markets_skips_closed_and_mismatched_markets(): + payload = [ + { + "id": "1", + "conditionId": "0x" + "a" * 64, + "clobTokenIds": '["1"]', + "outcomes": '["Yes", "No"]', + "active": True, + "closed": False, + "acceptingOrders": True, + }, + { + "id": "2", + "conditionId": "0x" + "b" * 64, + "clobTokenIds": '["2", "3"]', + "outcomes": '["Yes", "No"]', + "active": False, + "closed": True, + }, + ] + + assert normalize_markets(payload, 5) == [] + + +def test_normalize_markets_rejects_malformed_external_identifiers(): + payload = [ + { + "id": "1", + "conditionId": "not-a-condition-hash", + "clobTokenIds": '["1", "2"]', + "outcomes": '["Yes", "No"]', + "active": True, + "closed": False, + "acceptingOrders": True, + }, + { + "id": "2", + "conditionId": "0x" + "b" * 64, + "clobTokenIds": '["3", "4"]', + "outcomes": '["Yes", "No"]', + "active": True, + "closed": False, + "acceptingOrders": True, + }, + ] + + result = normalize_markets(payload, 5) + + assert {row.market_id for row in result} == {2} + + +def test_normalize_markets_skips_markets_without_live_order_books(): + payload = [ + { + "id": "1", + "conditionId": "0x" + "a" * 64, + "clobTokenIds": '["1", "2"]', + "outcomes": '["Yes", "No"]', + "active": True, + "closed": False, + "acceptingOrders": False, + } + ] + + assert normalize_markets(payload, 5) == [] + + +@pytest.mark.parametrize( + ("message", "kind", "midpoint"), + [ + ( + { + "event_type": "book", + "market": "0x" + "1" * 64, + "asset_id": "123", + "timestamp": "1782753357257", + "bids": [{"price": "0.48", "size": "2"}], + "asks": [{"price": "0.52", "size": "3"}], + }, + "book_snapshot", + "0.5", + ), + ( + { + "event_type": "last_trade_price", + "market": "0x" + "1" * 64, + "asset_id": "123", + "timestamp": "1782753357257", + "price": "0.51", + "size": "9", + "side": "BUY", + }, + "last_trade_price", + "0", + ), + ( + { + "event_type": "best_bid_ask", + "market": "0x" + "1" * 64, + "asset_id": "123", + "timestamp": "1782753357257", + "best_bid": "0.2", + "best_ask": "0.4", + }, + "best_bid_ask", + "0.3", + ), + ], +) +def test_normalize_ws_event_types(message, kind, midpoint): + row = normalize_ws_message(message)[0] + + assert row["event_kind"] == kind + assert row["midpoint"] == midpoint + assert len(row["event_id"]) == 64 + + +def test_normalize_wrapped_price_changes_flattens_each_change(): + message = { + "topic": "market", + "type": "price_change", + "payload": { + "market": "0x" + "2" * 64, + "timestamp": "1782753357257", + "priceChanges": [ + { + "tokenId": "10", + "price": "0.1", + "size": "2", + "side": "BUY", + "bestBid": "0.1", + "bestAsk": "0.2", + }, + { + "tokenId": "11", + "price": "0.9", + "size": "3", + "side": "SELL", + "bestBid": "0.8", + "bestAsk": "0.9", + }, + ], + }, + } + + rows = normalize_ws_message(message) + + assert [row["token_id"] for row in rows] == [10, 11] + assert [row["midpoint"] for row in rows] == ["0.15", "0.85"] + assert json.loads(rows[0]["raw_payload"])["change"]["tokenId"] == "10" + assert "priceChanges" not in json.loads(rows[0]["raw_payload"]) + + +def test_normalize_book_uses_highest_bid_and_lowest_ask(): + row = normalize_book( + { + "asset_id": "9", + "timestamp": "1782753357257", + "bids": [{"price": "0.2"}, {"price": "0.3"}], + "asks": [{"price": "0.7"}, {"price": "0.6"}], + }, + "0x" + "3" * 64, + ) + + assert row["best_bid"] == "0.3" + assert row["best_ask"] == "0.6" + assert row["midpoint"] == "0.45" + assert row["source"] == "CLOB_REST" + + +def test_trade_id_is_deterministic_and_uses_full_row_identity(): + base = { + "transactionHash": "0x" + "4" * 64, + "asset": "123", + "proxyWallet": "0x" + "5" * 40, + "side": "BUY", + "price": "0.5000", + "size": "2.00", + "timestamp": 1_700_000_000, + "conditionId": "0x" + "6" * 64, + "outcome": "Yes", + "title": "Question", + } + + first = normalize_trade(base) + second = normalize_trade({**base, "price": "0.5"}) + different = normalize_trade({**base, "size": "3"}) + + assert first["trade_id"] == second["trade_id"] + assert first["trade_id"] != different["trade_id"] + assert first["event_at"] == datetime.fromtimestamp(1_700_000_000, UTC) + + +def test_trade_rejects_malformed_fixed_width_identifiers(): + with pytest.raises(ValueError, match="proxyWallet"): + normalize_trade( + { + "transactionHash": "0x" + "4" * 64, + "asset": "123", + "proxyWallet": "not-a-wallet", + "price": "0.5", + "size": "2", + "timestamp": 1_700_000_000, + "conditionId": "0x" + "6" * 64, + } + ) + + +def test_stable_batch_token_is_order_independent(): + assert stable_batch_token(["b", "a"]) == stable_batch_token(["a", "b"]) + + +def test_decimal_text_rejects_invalid_input(): + with pytest.raises(ValueError, match="invalid decimal"): + decimal_text("not-a-number") + + +@pytest.mark.parametrize("value", ["Infinity", "NaN", "1e999999999", "9" * 129]) +def test_decimal_text_rejects_non_finite_or_unbounded_input(value): + with pytest.raises(ValueError): + decimal_text(value) diff --git a/workshops/build_workshop/polymarket/collector/tests/test_service.py b/workshops/build_workshop/polymarket/collector/tests/test_service.py new file mode 100644 index 0000000..ed67f83 --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/tests/test_service.py @@ -0,0 +1,433 @@ +import asyncio +from dataclasses import replace +from datetime import UTC, datetime, timedelta + +import aiohttp +import pytest + +from collector.config import Settings +from collector.health import HealthState +from collector.service import CollectorService, DedupeWindow, fixture_markets, fixture_ticks + + +class FakeAPI: + async def discover(self, count): + return fixture_markets(count) + + +class FakeStorage: + def __init__(self, failures=0): + self.failures = failures + self.calls = [] + self.successful_write = asyncio.Event() + + async def ping(self): + return None + + async def assert_schema(self): + return None + + async def recent_ids(self, _minutes): + return {"already-seen"} + + async def trade_checkpoints(self, _conditions): + return {} + + async def insert(self, table, rows, columns, token): + self.calls.append((table, rows, columns, token)) + if self.failures: + self.failures -= 1 + raise RuntimeError("temporary write failure") + self.successful_write.set() + + +def settings(mode="fixture"): + return Settings( + clickhouse_host="example.clickhouse.cloud", + clickhouse_port=8443, + clickhouse_user="default", + clickhouse_password="secret", + clickhouse_database="polymarket", + clickhouse_secure=True, + mode=mode, + market_count=2, + reconcile_seconds=1, + book_fallback_seconds=1, + stall_seconds=1, + dedupe_minutes=15, + initial_lookback_minutes=10, + health_port=8090, + ) + + +def test_dedupe_window_reserves_commits_releases_and_evicts(): + window = DedupeWindow(capacity=2) + window.hydrate({"old"}) + + assert window.reserve(["old", "new"]) == ["new"] + assert window.reserve(["new"]) == [] + window.release(["new"]) + assert window.reserve(["new"]) == ["new"] + window.commit(["new"]) + window.commit(["third"]) + + assert "old" not in window.committed + assert set(window.committed) == {"new", "third"} + + +def test_dedupe_hydration_keeps_newest_ids_at_the_lru_tail(): + window = DedupeWindow(capacity=2) + window.hydrate(["older", "newer"]) + + window.commit(["newest"]) + + assert list(window.committed) == ["newer", "newest"] + + +@pytest.mark.asyncio +async def test_prepare_hydrates_dedupe_and_persists_fixture_markets(): + storage = FakeStorage() + health = HealthState() + service = CollectorService(settings(), FakeAPI(), storage, health) + + await service.prepare() + + assert "already-seen" in service.dedupe.committed + assert storage.calls[0][0] == "markets" + assert health.status == "fixture" + assert health.watched_markets == 2 + + +@pytest.mark.asyncio +async def test_enqueue_filters_committed_and_pending_ids(): + storage = FakeStorage() + service = CollectorService(settings(), FakeAPI(), storage, HealthState()) + service.dedupe.hydrate({"seen"}) + rows = [ + {"event_id": "seen", "value": 1}, + {"event_id": "fresh", "value": 2}, + {"event_id": "fresh", "value": 2}, + ] + + await service.enqueue("price_ticks", rows, ["event_id", "value"], "event_id") + + batch = service.queue.get_nowait() + assert [row["event_id"] for row in batch.rows] == ["fresh"] + assert batch.ids == ["fresh"] + + +@pytest.mark.asyncio +async def test_writer_retries_identical_batch_before_committing_id(): + storage = FakeStorage(failures=1) + service = CollectorService( + settings(), FakeAPI(), storage, HealthState(), sleep=lambda _: asyncio.sleep(0) + ) + rows = fixture_ticks(fixture_markets(1), 1, datetime.now(UTC)) + await service.enqueue("price_ticks", rows, list(rows[0]), "event_id") + + task = asyncio.create_task(service.writer_loop()) + await asyncio.wait_for(storage.successful_write.wait(), timeout=1) + await asyncio.wait_for(service.queue.join(), timeout=1) + service.stop.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + assert len(storage.calls) == 2 + assert storage.calls[0][3] == storage.calls[1][3] + assert rows[0]["event_id"] in service.dedupe.committed + assert service.health.last_clickhouse_write_at is not None + + +@pytest.mark.asyncio +async def test_writer_coalesces_compatible_small_batches(): + storage = FakeStorage() + service = CollectorService(settings(), FakeAPI(), storage, HealthState()) + await service.enqueue( + "price_ticks", [{"event_id": "one"}], ["event_id"], "event_id" + ) + await service.enqueue( + "price_ticks", [{"event_id": "two"}], ["event_id"], "event_id" + ) + + task = asyncio.create_task(service.writer_loop()) + await asyncio.wait_for(storage.successful_write.wait(), timeout=1) + await asyncio.wait_for(service.queue.join(), timeout=1) + service.stop.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + assert len(storage.calls) == 1 + assert [row["event_id"] for row in storage.calls[0][1]] == ["one", "two"] + + +@pytest.mark.asyncio +async def test_queue_capacity_counts_rows_and_applies_backpressure(): + constrained = replace(settings(), queue_capacity=2) + service = CollectorService(constrained, FakeAPI(), FakeStorage(), HealthState()) + first_rows = [ + {"event_id": "first", "value": 1}, + {"event_id": "second", "value": 2}, + ] + await service.enqueue( + "price_ticks", first_rows, ["event_id", "value"], "event_id" + ) + + blocked = asyncio.create_task( + service.enqueue( + "price_ticks", + [{"event_id": "third", "value": 3}], + ["event_id", "value"], + "event_id", + ) + ) + await asyncio.sleep(0) + + assert service.health.queue_depth == 2 + assert service.health.status == "unhealthy" + assert blocked.done() is False + + first_batch = service.queue.get_nowait() + await service._release_capacity(len(first_batch.rows)) + await blocked + + assert service.health.queue_depth == 1 + assert service.queue.get_nowait().ids == ["third"] + + +@pytest.mark.asyncio +async def test_reconciled_checkpoint_advances_when_every_row_is_duplicate(): + service = CollectorService(settings(), FakeAPI(), FakeStorage(), HealthState()) + checkpoint = datetime.now(UTC) + service.dedupe.hydrate({"seen"}) + + await service.enqueue( + "trades", + [{"trade_id": "seen"}], + ["trade_id"], + "trade_id", + {"condition": checkpoint}, + ) + + assert service.queue.empty() + assert service.checkpoints["condition"] == checkpoint + + +@pytest.mark.asyncio +async def test_empty_reconciliation_advances_checkpoint_without_a_write(): + service = CollectorService(settings(), FakeAPI(), FakeStorage(), HealthState()) + checkpoint = datetime.now(UTC) + + await service.enqueue( + "trades", [], ["trade_id"], "trade_id", {"condition": checkpoint} + ) + + assert service.queue.empty() + assert service.checkpoints["condition"] == checkpoint + + +def test_source_rows_must_match_the_discovered_token_condition_pair(): + health = HealthState() + service = CollectorService(settings(), FakeAPI(), FakeStorage(), health) + service.tokens = fixture_markets(1) + token = service.tokens[0] + valid = {"token_id": token.token_id, "condition_id": token.condition_id} + wrong_condition = {**valid, "condition_id": "0x" + "9" * 64} + unknown_token = {**valid, "token_id": token.token_id + 99} + + accepted = service._filter_watched( + [valid, wrong_condition, unknown_token], "test_source" + ) + + assert accepted == [valid] + assert health.source_parse_errors_total == 2 + + +def test_non_midpoint_events_do_not_claim_quote_freshness(): + service = CollectorService(settings("live"), FakeAPI(), FakeStorage(), HealthState()) + service.tokens = fixture_markets(1) + token = service.tokens[0] + + accepted = service._filter_watched( + [ + { + "token_id": token.token_id, + "condition_id": token.condition_id, + "midpoint": "0", + } + ], + "websocket", + ) + + assert len(accepted) == 1 + assert service._token_price_at == {} + + +def test_source_health_recovers_and_clickhouse_failure_has_precedence(): + now = datetime.now(UTC) + health = HealthState(started_at=now - timedelta(minutes=2)) + service = CollectorService(settings("live"), FakeAPI(), FakeStorage(), health) + service.tokens = fixture_markets(1) + + service._refresh_health(now) + assert health.status == "unhealthy" + assert health.reason == "all_sources_stale" + + service._ws_healthy = True + health.last_websocket_event_at = now + health.last_trade_reconcile_at = now + for token in service.tokens: + service._token_price_at[token.token_id] = now + service._token_websocket_at[token.token_id] = now + health.watched_tokens = len(service.tokens) + service._refresh_health(now) + assert health.status == "live" + + service._write_failure_started_at = now - timedelta(seconds=61) + service._refresh_health(now) + assert health.status == "unhealthy" + assert health.reason == "clickhouse_write_stalled" + + +@pytest.mark.asyncio +async def test_websocket_loop_retries_transport_failure_and_enqueues_data(): + class ReconnectingAPI: + def __init__(self): + self.calls = 0 + + async def websocket_messages(self, _token_ids, _stall_seconds): + self.calls += 1 + if self.calls == 1: + raise aiohttp.ClientConnectionError("handshake failed") + token = fixture_markets(1)[0] + yield { + "event_type": "best_bid_ask", + "market": token.condition_id, + "asset_id": str(token.token_id), + "timestamp": "1782753357257", + "best_bid": "0.4", + "best_ask": "0.6", + } + await asyncio.sleep(3600) + + api = ReconnectingAPI() + service = CollectorService( + settings("live"), api, FakeStorage(), HealthState(), sleep=lambda _: asyncio.sleep(0) + ) + service.tokens = fixture_markets(1) + task = asyncio.create_task(service.websocket_loop()) + + batch = await asyncio.wait_for(service.queue.get(), timeout=1) + service.stop.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + assert api.calls >= 2 + assert batch.rows[0]["source"] == "WEBSOCKET" + assert service.health.source_errors_total == 1 + + +@pytest.mark.asyncio +async def test_book_fallback_keeps_ticks_flowing_while_websocket_is_down(): + class BookAPI: + async def book(self, token_id): + return { + "asset_id": str(token_id), + "timestamp": "1782753357257", + "bids": [{"price": "0.4"}], + "asks": [{"price": "0.6"}], + } + + service = CollectorService( + settings("live"), + BookAPI(), + FakeStorage(), + HealthState(), + sleep=lambda _: asyncio.sleep(0.001), + ) + service.tokens = fixture_markets(1) + task = asyncio.create_task(service.book_fallback_loop()) + + batch = await asyncio.wait_for(service.queue.get(), timeout=1) + service.stop.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + assert all(row["source"] == "CLOB_REST" for row in batch.rows) + assert service.health.last_book_fallback_at is not None + + +@pytest.mark.asyncio +async def test_prepare_no_markets_sets_degraded(monkeypatch): + class EmptyAPI: + async def discover(self, _count): + return [] + + storage = FakeStorage() + health = HealthState() + service = CollectorService(settings("live"), EmptyAPI(), storage, health) + + await service.prepare() + + assert health.status == "degraded" + assert health.reason == "no_active_markets" + assert storage.calls == [] + + +@pytest.mark.asyncio +async def test_run_surfaces_an_unexpected_background_task_failure(): + service = CollectorService(settings(), FakeAPI(), FakeStorage(), HealthState()) + + async def fail_fixture_loop(): + raise RuntimeError("fixture task crashed") + + service.fixture_loop = fail_fixture_loop + + with pytest.raises(RuntimeError, match="fixture task crashed"): + await service.run() + + +@pytest.mark.asyncio +async def test_all_malformed_trade_page_does_not_advance_checkpoint(): + class MalformedAPI: + async def trades(self, _condition_id, _start, _end): + return [{"conditionId": "bad"}] + + service = CollectorService(settings("live"), MalformedAPI(), FakeStorage(), HealthState()) + service.tokens = fixture_markets(1) + condition_id = service.tokens[0].condition_id + + succeeded = await service._reconcile_condition(condition_id, datetime.now(UTC)) + + assert succeeded is False + assert condition_id not in service.checkpoints + + +@pytest.mark.asyncio +async def test_trade_reconciliation_rejects_only_the_malformed_item(): + class MixedAPI: + async def trades(self, _condition_id, _start, _end): + return [ + {"conditionId": "missing-required-fields"}, + { + "conditionId": "0x" + f"{1:064x}", + "asset": "10000000", + "timestamp": 1_700_000_000, + "transactionHash": "0x" + "2" * 64, + "proxyWallet": "0x" + "3" * 40, + "side": "BUY", + "price": "0.5", + "size": "2", + }, + ] + + service = CollectorService(settings("live"), MixedAPI(), FakeStorage(), HealthState()) + service.tokens = fixture_markets(1) + + task = asyncio.create_task(service.trade_reconcile_loop()) + batch = await asyncio.wait_for(service.queue.get(), timeout=1) + service.stop.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + assert len(batch.rows) == 1 + assert service.health.source_parse_errors_total == 1 diff --git a/workshops/build_workshop/polymarket/collector/tests/test_storage_integration.py b/workshops/build_workshop/polymarket/collector/tests/test_storage_integration.py new file mode 100644 index 0000000..20ce3ef --- /dev/null +++ b/workshops/build_workshop/polymarket/collector/tests/test_storage_integration.py @@ -0,0 +1,131 @@ +import asyncio +import os +from datetime import UTC, datetime + +import pytest + +from collector.config import Settings +from collector.models import build_tick, normalize_trade, stable_batch_token +from collector.storage import MARKET_COLUMNS, TICK_COLUMNS, TRADE_COLUMNS, ClickHouseStorage + + +pytestmark = pytest.mark.skipif( + os.getenv("POLYMARKET_CLICKHOUSE_INTEGRATION") != "1", + reason="requires the disposable ClickHouse integration container", +) + + +@pytest.mark.asyncio +async def test_clickhouse_storage_round_trip_and_retry_deduplication(): + settings = Settings( + clickhouse_host=os.environ["CLICKHOUSE_HOST"], + clickhouse_port=int(os.environ["CLICKHOUSE_PORT"]), + clickhouse_user="default", + clickhouse_password=os.environ["CLICKHOUSE_PASSWORD"], + clickhouse_database="polymarket", + clickhouse_secure=False, + mode="fixture", + market_count=1, + reconcile_seconds=10, + book_fallback_seconds=30, + stall_seconds=30, + dedupe_minutes=15, + initial_lookback_minutes=10, + health_port=8090, + ) + storage = ClickHouseStorage(settings) + now = datetime.now(UTC) + condition_id = "0x" + "a" * 64 + token_id = 123456789 + market = { + "market_id": 1, + "condition_id": condition_id, + "token_id": token_id, + "outcome": "Yes", + "question": "Will the adapter integration pass?", + "slug": "adapter-integration", + "active": True, + "accepting_orders": False, + "volume_24h": "100", + "observed_at": now, + } + tick = build_tick( + event_kind="best_bid_ask", + source="FIXTURE", + condition_id=condition_id, + token_id=token_id, + timestamp=now.isoformat(), + best_bid="0.48", + best_ask="0.52", + source_hash="adapter-integration", + raw={"fixture": True}, + ) + trade = normalize_trade( + { + "transactionHash": "0x" + "b" * 64, + "asset": str(token_id), + "proxyWallet": "0x" + "c" * 40, + "side": "BUY", + "price": "0.5", + "size": "2", + "timestamp": int(now.timestamp()), + "conditionId": condition_id, + "outcome": "Yes", + "title": market["question"], + } + ) + + try: + await storage.ping() + await storage.assert_schema() + await storage.insert("markets", [market], MARKET_COLUMNS, "market-adapter") + tick_token = stable_batch_token([tick["event_id"]]) + await storage.insert("price_ticks", [tick], TICK_COLUMNS, tick_token) + await storage.insert("price_ticks", [tick], TICK_COLUMNS, tick_token) + await storage.insert( + "trades", [trade], TRADE_COLUMNS, stable_batch_token([trade["trade_id"]]) + ) + + recent_ids = await storage.recent_ids(15) + checkpoints = await storage.trade_checkpoints([condition_id]) + + def counts(): + return storage.client.query( + """ + SELECT + ( + SELECT count() + FROM polymarket.price_ticks + WHERE event_id = {tick_id:String} + ), + ( + SELECT count() + FROM polymarket.trades_clean + WHERE trade_id = {trade_id:String} + ), + ( + SELECT countMerge(updates) + FROM polymarket.market_midpoints_1m + WHERE token_id = {token_id:UInt256} + ) + """, + parameters={ + "tick_id": tick["event_id"], + "trade_id": trade["trade_id"], + "token_id": token_id, + }, + ).first_row + + tick_count, trade_count, candle_count = await asyncio.to_thread(counts) + # The standalone MergeTree test server has no Keeper-backed insert + # deduplication. ClickHouse Cloud's SharedMergeTree applies the token; + # this check proves the real adapter accepts and retries that setting. + assert tick_count == 2 + assert trade_count == 1 + assert candle_count == 2 + assert {tick["event_id"], trade["trade_id"]} <= set(recent_ids) + assert checkpoints[condition_id].replace(microsecond=0) == now.replace( + microsecond=0 + ) + finally: + await storage.close() diff --git a/workshops/build_workshop/polymarket/db/queries.sql b/workshops/build_workshop/polymarket/db/queries.sql new file mode 100644 index 0000000..e72938b --- /dev/null +++ b/workshops/build_workshop/polymarket/db/queries.sql @@ -0,0 +1,92 @@ +-- 1. Current probability and freshness +SELECT + m.token_id, + m.question, + m.outcome, + round(argMax(t.midpoint, t.event_at) * 100, 2) AS probability_percent, + max(t.event_at) AS last_update +FROM polymarket.price_ticks AS t +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = t.token_id +WHERE t.midpoint > 0 + AND t.event_at >= now() - INTERVAL 30 MINUTE +GROUP BY m.token_id, m.question, m.outcome +ORDER BY m.question, m.outcome; + +-- 2. Five-minute movers +WITH now() AS current_time +SELECT + m.token_id, + m.question, + m.outcome, + round(argMaxIf(t.midpoint, t.event_at, t.event_at > current_time - INTERVAL 1 MINUTE) * 100, 2) AS now_percent, + round(argMaxIf(t.midpoint, t.event_at, t.event_at <= current_time - INTERVAL 5 MINUTE) * 100, 2) AS five_minutes_ago_percent, + round(now_percent - five_minutes_ago_percent, 2) AS move_points +FROM polymarket.price_ticks AS t +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = t.token_id +WHERE t.midpoint > 0 + AND t.event_at >= current_time - INTERVAL 15 MINUTE +GROUP BY m.token_id, m.question, m.outcome +HAVING now_percent > 0 AND five_minutes_ago_percent > 0 +ORDER BY abs(move_points) DESC; + +-- 3. Spread and freshness +SELECT + m.token_id, + m.question, + m.outcome, + round(argMax(t.best_bid, t.event_at) * 100, 2) AS bid_percent, + round(argMax(t.best_ask, t.event_at) * 100, 2) AS ask_percent, + round(ask_percent - bid_percent, 2) AS spread_points, + dateDiff('second', max(t.event_at), now()) AS age_seconds +FROM polymarket.price_ticks AS t +INNER JOIN +( + SELECT token_id, question, outcome + FROM polymarket.markets FINAL +) AS m ON m.token_id = t.token_id +WHERE t.best_bid > 0 + AND t.best_ask > 0 + AND t.event_at >= now() - INTERVAL 30 MINUTE +GROUP BY m.token_id, m.question, m.outcome +ORDER BY spread_points DESC; + +-- 4. Trade-volume velocity +SELECT + condition_id, + token_id, + title, + outcome, + round(sumIf(price * size, event_at >= now() - INTERVAL 5 MINUTE), 2) AS current_5m_usd, + round(sumIf( + price * size, + event_at >= now() - INTERVAL 10 MINUTE + AND event_at < now() - INTERVAL 5 MINUTE + ), 2) AS previous_5m_usd, + round(current_5m_usd / greatest(previous_5m_usd, 0.01), 2) AS velocity_ratio +FROM polymarket.trades_clean +WHERE event_at >= now() - INTERVAL 10 MINUTE +GROUP BY condition_id, token_id, title, outcome +ORDER BY current_5m_usd DESC; + +-- 5. One-minute quote-midpoint OHLC +SELECT + minute, + token_id, + round(argMinMerge(open) * 100, 2) AS open_percent, + round(maxMerge(high) * 100, 2) AS high_percent, + round(minMerge(low) * 100, 2) AS low_percent, + round(argMaxMerge(close) * 100, 2) AS close_percent, + countMerge(updates) AS updates +FROM polymarket.market_midpoints_1m +WHERE minute >= now() - INTERVAL 30 MINUTE +GROUP BY minute, token_id +ORDER BY minute DESC, token_id +LIMIT 30; diff --git a/workshops/build_workshop/polymarket/db/schema.sql b/workshops/build_workshop/polymarket/db/schema.sql new file mode 100644 index 0000000..b36efe0 --- /dev/null +++ b/workshops/build_workshop/polymarket/db/schema.sql @@ -0,0 +1,95 @@ +CREATE DATABASE IF NOT EXISTS polymarket; + +CREATE TABLE IF NOT EXISTS polymarket.markets +( + market_id UInt64, + condition_id FixedString(66), + token_id UInt256, + outcome LowCardinality(String), + question String, + slug String, + active Bool, + accepting_orders Bool, + volume_24h Decimal128(8), + observed_at DateTime64(3, 'UTC') +) +ENGINE = ReplacingMergeTree(observed_at) +ORDER BY (condition_id, token_id); + +CREATE TABLE IF NOT EXISTS polymarket.price_ticks +( + event_id FixedString(64), + condition_id FixedString(66), + token_id UInt256, + event_at DateTime64(3, 'UTC'), + observed_at DateTime64(3, 'UTC'), + event_kind Enum8( + 'book_snapshot' = 1, + 'price_change' = 2, + 'last_trade_price' = 3, + 'best_bid_ask' = 4, + 'rest_book' = 5 + ), + source Enum8('WEBSOCKET' = 1, 'CLOB_REST' = 2, 'FIXTURE' = 3), + price Decimal64(12), + size Decimal128(8), + side Enum8('UNKNOWN' = 0, 'BUY' = 1, 'SELL' = 2), + best_bid Decimal64(12), + best_ask Decimal64(12), + midpoint Decimal64(12), + source_hash String, + raw_payload String +) +ENGINE = MergeTree +ORDER BY (toStartOfHour(event_at), token_id, event_at, event_id); + +CREATE TABLE IF NOT EXISTS polymarket.trades +( + trade_id FixedString(64), + condition_id FixedString(66), + token_id UInt256, + event_at DateTime64(3, 'UTC'), + observed_at DateTime64(3, 'UTC'), + proxy_wallet FixedString(42), + side Enum8('UNKNOWN' = 0, 'BUY' = 1, 'SELL' = 2), + price Decimal64(12), + size Decimal128(8), + outcome LowCardinality(String), + transaction_hash FixedString(66), + title String +) +ENGINE = ReplacingMergeTree(observed_at) +ORDER BY (toStartOfHour(event_at), condition_id, event_at, trade_id); + +CREATE OR REPLACE VIEW polymarket.trades_clean AS +SELECT * +FROM polymarket.trades FINAL; + +CREATE TABLE IF NOT EXISTS polymarket.market_midpoints_1m +( + token_id UInt256, + minute DateTime('UTC'), + open AggregateFunction(argMin, Decimal64(12), Tuple(DateTime64(3, 'UTC'), FixedString(64))), + high AggregateFunction(max, Decimal64(12)), + low AggregateFunction(min, Decimal64(12)), + close AggregateFunction(argMax, Decimal64(12), Tuple(DateTime64(3, 'UTC'), FixedString(64))), + updates AggregateFunction(count) +) +ENGINE = AggregatingMergeTree +ORDER BY (minute, token_id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS polymarket.market_midpoints_1m_mv +TO polymarket.market_midpoints_1m +AS +SELECT + token_id, + toStartOfMinute(event_at) AS minute, + argMinState(midpoint, tuple(event_at, event_id)) AS open, + maxState(midpoint) AS high, + minState(midpoint) AS low, + argMaxState(midpoint, tuple(event_at, event_id)) AS close, + countState() AS updates +FROM polymarket.price_ticks +WHERE midpoint > 0 + AND event_kind IN ('book_snapshot', 'price_change', 'best_bid_ask', 'rest_book') +GROUP BY token_id, minute; diff --git a/workshops/build_workshop/polymarket/docker-compose.yml b/workshops/build_workshop/polymarket/docker-compose.yml new file mode 100644 index 0000000..c24d6e4 --- /dev/null +++ b/workshops/build_workshop/polymarket/docker-compose.yml @@ -0,0 +1,38 @@ +services: + collector: + build: + context: ./collector + container_name: polymarket-workshop-collector + environment: + CLICKHOUSE_HOST: ${CLICKHOUSE_HOST:-} + CLICKHOUSE_PORT: ${CLICKHOUSE_PORT:-8443} + CLICKHOUSE_USER: ${CLICKHOUSE_USER:-default} + CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:-} + CLICKHOUSE_DATABASE: ${CLICKHOUSE_DATABASE:-polymarket} + CLICKHOUSE_SECURE: ${CLICKHOUSE_SECURE:-true} + POLYMARKET_MODE: ${POLYMARKET_MODE:-live} + POLYMARKET_MARKET_COUNT: ${POLYMARKET_MARKET_COUNT:-5} + POLYMARKET_RECONCILE_SECONDS: ${POLYMARKET_RECONCILE_SECONDS:-10} + POLYMARKET_BOOK_FALLBACK_SECONDS: ${POLYMARKET_BOOK_FALLBACK_SECONDS:-30} + POLYMARKET_STALL_SECONDS: ${POLYMARKET_STALL_SECONDS:-30} + POLYMARKET_DEDUPE_MINUTES: ${POLYMARKET_DEDUPE_MINUTES:-15} + POLYMARKET_INITIAL_LOOKBACK_MINUTES: ${POLYMARKET_INITIAL_LOOKBACK_MINUTES:-10} + POLYMARKET_HEALTH_PORT: ${POLYMARKET_HEALTH_PORT:-8090} + ports: + - "127.0.0.1:${POLYMARKET_HEALTH_PORT:-8090}:${POLYMARKET_HEALTH_PORT:-8090}" + restart: unless-stopped + stop_grace_period: 60s + healthcheck: + test: + - CMD + - python + - -c + - >- + import json, os, urllib.request; + port=os.environ.get('POLYMARKET_HEALTH_PORT','8090'); + data=json.load(urllib.request.urlopen(f'http://127.0.0.1:{port}/health', timeout=3)); + assert data['status'] in {'live','degraded','fixture'} + interval: 10s + timeout: 5s + retries: 6 + start_period: 20s diff --git a/workshops/build_workshop/polymarket/preflight.sh b/workshops/build_workshop/polymarket/preflight.sh new file mode 100755 index 0000000..d42b229 --- /dev/null +++ b/workshops/build_workshop/polymarket/preflight.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ENV_FILE="${ROOT}/.env.polymarket" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +command -v docker >/dev/null 2>&1 || fail "Docker is not installed" +docker compose version >/dev/null 2>&1 || fail "Docker Compose v2 is not available" +command -v clickhouse >/dev/null 2>&1 || fail "ClickHouse client is not installed" + +[[ -f "${ENV_FILE}" ]] || fail "copy .env.polymarket.example to .env.polymarket first" +set -a +# shellcheck disable=SC1090 +source "${ENV_FILE}" +set +a + +[[ -n "${CLICKHOUSE_HOST:-}" ]] || fail "CLICKHOUSE_HOST is empty" +[[ -n "${CLICKHOUSE_PASSWORD:-}" ]] || fail "CLICKHOUSE_PASSWORD is empty" +if [[ "${CLICKHOUSE_HOST}" =~ ^(localhost|127\.0\.0\.1|::1)$ ]]; then + fail "CLICKHOUSE_HOST points to a local server; this track uses ClickHouse Cloud" +fi + +clickhouse client \ + --host "${CLICKHOUSE_HOST}" \ + --port "${CLICKHOUSE_PORT:-8443}" \ + --user "${CLICKHOUSE_USER:-default}" \ + --password "${CLICKHOUSE_PASSWORD}" \ + --secure \ + --query "SELECT 1" >/dev/null + +public_ok=true +market_payload=$( + curl --fail --silent --show-error --max-time 15 \ + 'https://gamma-api.polymarket.com/markets?active=true&closed=false&limit=20' +) || public_ok=false + +condition_id="" +token_id="" +if [[ "${public_ok}" == "true" ]]; then + if ! read -r condition_id token_id < <( + printf '%s' "${market_payload}" | python3 -c ' +import json, sys +for market in json.load(sys.stdin): + if market.get("acceptingOrders"): + tokens = json.loads(market["clobTokenIds"]) + if tokens: + print(market["conditionId"], tokens[0]) + break +' + ); then + public_ok=false + fi + [[ -n "${condition_id}" && -n "${token_id}" ]] || public_ok=false +fi + +if [[ "${public_ok}" == "true" ]]; then + curl --fail --silent --show-error --max-time 15 --get \ + 'https://data-api.polymarket.com/trades' \ + --data-urlencode "market=${condition_id}" \ + --data-urlencode 'limit=1' >/dev/null || public_ok=false + curl --fail --silent --show-error --max-time 15 --get \ + 'https://clob.polymarket.com/book' \ + --data-urlencode "token_id=${token_id}" >/dev/null || public_ok=false +fi + +if [[ "${public_ok}" != "true" ]]; then + if [[ "${POLYMARKET_MODE:-live}" == "fixture" ]]; then + echo "WARN: Polymarket public data is unreachable; fixture mode will continue" + else + fail "a Polymarket public API is unreachable; set POLYMARKET_MODE=fixture and rerun" + fi +fi + +if [[ "${public_ok}" == "true" ]]; then + echo "READY: Docker, ClickHouse Cloud, Gamma, Data API, and CLOB REST are reachable" +else + echo "READY: Docker and ClickHouse Cloud are reachable; fixture mode will supply data" +fi diff --git a/workshops/build_workshop/polymarket/test-clickhouse.sh b/workshops/build_workshop/polymarket/test-clickhouse.sh new file mode 100755 index 0000000..abbe60e --- /dev/null +++ b/workshops/build_workshop/polymarket/test-clickhouse.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +NAME="polymarket-workshop-test-${RANDOM}" + +cleanup() { + docker rm -f "${NAME}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker run -d \ + --name "${NAME}" \ + -e CLICKHOUSE_PASSWORD=workshop \ + -p 127.0.0.1::8123 \ + clickhouse/clickhouse-server:26.3 >/dev/null +for _ in $(seq 1 60); do + if docker exec "${NAME}" clickhouse client --password workshop --query "SELECT 1" >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker exec "${NAME}" clickhouse client --password workshop --query "SELECT 1" >/dev/null +docker exec -i "${NAME}" clickhouse client --password workshop --multiquery < "${ROOT}/db/schema.sql" + +docker exec -i "${NAME}" clickhouse client --password workshop --multiquery <<'SQL' +INSERT INTO polymarket.markets VALUES +( + 1, + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1001, + 'Yes', + 'Will the fixture move?', + 'fixture-move', + true, + false, + 1000, + now64(3) +); + +INSERT INTO polymarket.price_ticks VALUES +( + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1001, + now64(3) - INTERVAL 1 MINUTE, + now64(3), + 'best_bid_ask', + 'FIXTURE', + 0, + 0, + 'UNKNOWN', + 0.48, + 0.52, + 0.50, + 'fixture-1', + '{}' +), +( + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1001, + now64(3), + now64(3), + 'best_bid_ask', + 'FIXTURE', + 0, + 0, + 'UNKNOWN', + 0.53, + 0.57, + 0.55, + 'fixture-2', + '{}' +); + +INSERT INTO polymarket.trades VALUES +( + 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1001, + toDateTime64('2026-07-29 00:00:00', 3, 'UTC'), + now64(3), + '0x1111111111111111111111111111111111111111', + 'BUY', + 0.55, + 10, + 'Yes', + '0x2222222222222222222222222222222222222222222222222222222222222222', + 'Will the fixture move?' +), +( + 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + '0x1111111111111111111111111111111111111111111111111111111111111111', + 1001, + toDateTime64('2026-07-29 00:00:00', 3, 'UTC'), + now64(3) + INTERVAL 1 SECOND, + '0x1111111111111111111111111111111111111111', + 'BUY', + 0.56, + 10, + 'Yes', + '0x2222222222222222222222222222222222222222222222222222222222222222', + 'Will the fixture move?' +); +SQL + +for _ in $(seq 1 30); do + rows=$(docker exec "${NAME}" clickhouse client --password workshop --query \ + "SELECT count() FROM polymarket.market_midpoints_1m") + [[ "${rows}" -gt 0 ]] && break + sleep 1 +done + +docker exec "${NAME}" clickhouse client --password workshop --query \ + "SELECT throwIf(count() = 0, 'midpoint MV is empty') FROM polymarket.market_midpoints_1m" +docker exec "${NAME}" clickhouse client --password workshop --query \ + "SELECT throwIf(count() != 1, 'trade dedupe view failed') FROM polymarket.trades_clean" +docker exec "${NAME}" clickhouse client --password workshop --query \ + "SELECT throwIf(any(price) != 0.56, 'trade dedupe kept the older row') FROM polymarket.trades_clean" +docker exec "${NAME}" clickhouse client --password workshop --multiquery < "${ROOT}/db/queries.sql" >/dev/null +python3 "${ROOT}/../scripts/check-polymarket-sql.py" --print-selects \ + | docker exec -i "${NAME}" clickhouse client --password workshop --multiquery >/dev/null + +HOST_PORT=$(docker port "${NAME}" 8123/tcp | sed 's/.*://') +( + cd "${ROOT}/collector" + env \ + POLYMARKET_CLICKHOUSE_INTEGRATION=1 \ + CLICKHOUSE_HOST=127.0.0.1 \ + CLICKHOUSE_PORT="${HOST_PORT}" \ + CLICKHOUSE_PASSWORD=workshop \ + "${PYTHON_BIN:-python3}" -m pytest tests/test_storage_integration.py -q +) + +echo "Polymarket ClickHouse schema and reference queries passed" diff --git a/workshops/build_workshop/scripts/check-docs.sh b/workshops/build_workshop/scripts/check-docs.sh index 6aac6a2..32fdd94 100755 --- a/workshops/build_workshop/scripts/check-docs.sh +++ b/workshops/build_workshop/scripts/check-docs.sh @@ -27,7 +27,12 @@ fail_if_found \ fail_if_found \ "learners must not be told to edit SQL comments or execute a hidden SQL file" \ '(comment|uncomment).*(sql|variant)|(run|execute|open).*(db/cloud|\.sql file)' \ - "${CONTENT}/learner" + "${CONTENT}/learner" "${CONTENT}/polymarket/learner" + +fail_if_found \ + "the Polymarket track must not request trading credentials or call order endpoints" \ + 'POLYMARKET_(API_KEY|SECRET|PRIVATE_KEY)|/orders?([/?`[:space:]]|$)|wallet[_ -]?private' \ + "${CONTENT}/polymarket" "${ROOT}/polymarket" if grep -RInE --exclude='06b-ai-sre-librechat.mdx' \ '06a|06b|LibreChat' "${CONTENT}"; then @@ -87,6 +92,16 @@ require_fixed \ 'git switch build-workshop-v1' \ "${CONTENT}/learner/00-setup.mdx" +require_fixed \ + "Polymarket learner setup must switch to the production workshop branch" \ + 'git switch build-workshop-v1' \ + "${CONTENT}/polymarket/learner/00-setup.mdx" + +require_fixed \ + "Polymarket dev rehearsal must use the staging workshop branch" \ + 'git switch dev-build-workshop-v1' \ + "${CONTENT}/polymarket/rehearsal.mdx" + require_fixed \ "the former learner Module 06b page must remain clearly archived" \ 'Module 06b is no longer part of the workshop.' \ @@ -163,6 +178,34 @@ require_fixed \ 'POSTGRES = load_postgres_config()' \ "${ROOT}/app/loadgen/pg_trip_writer.py" +for literal in \ + 'CLICKHOUSE_HOST=' \ + 'CLICKHOUSE_PORT=8443' \ + 'CLICKHOUSE_DATABASE=polymarket' \ + 'POLYMARKET_MODE=live'; do + require_fixed \ + "the Polymarket environment must expose the documented Cloud/collector contract" \ + "${literal}" \ + "${ROOT}/polymarket/.env.polymarket.example" +done + +require_fixed \ + "Polymarket setup must source its environment after editing" \ + 'set -a; source ./.env.polymarket; set +a' \ + "${CONTENT}/polymarket/learner/00-setup.mdx" + +require_fixed \ + "Polymarket Module 02 must contain the complete copyable database DDL" \ + 'CREATE MATERIALIZED VIEW IF NOT EXISTS polymarket.market_midpoints_1m_mv' \ + "${CONTENT}/polymarket/learner/02-model-data.mdx" + +require_fixed \ + "Polymarket Module 05 must contain the clean trade-volume query" \ + 'FROM polymarket.trades_clean' \ + "${CONTENT}/polymarket/learner/05-investigate-movement.mdx" + +python3 "${ROOT}/scripts/check-polymarket-sql.py" + if command -v docker >/dev/null 2>&1; then compose_config=$( cd "${ROOT}/app" && @@ -175,6 +218,15 @@ if command -v docker >/dev/null 2>&1; then echo "ERROR: rendered workshop Compose config contains a local managed-service substitute" >&2 exit 1 fi + + polymarket_services=$( + cd "${ROOT}/polymarket" && + docker compose --env-file .env.polymarket.example config --services + ) + if [[ "${polymarket_services}" != "collector" ]]; then + echo "ERROR: Polymarket Compose must define only the stateless collector" >&2 + exit 1 + fi fi require_fixed \ diff --git a/workshops/build_workshop/scripts/check-polymarket-sql.py b/workshops/build_workshop/scripts/check-polymarket-sql.py new file mode 100644 index 0000000..53376e1 --- /dev/null +++ b/workshops/build_workshop/scripts/check-polymarket-sql.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Fail when executable Polymarket SQL drifts from learner copy blocks.""" + +from pathlib import Path +import re +import sys + + +ROOT = Path(__file__).resolve().parents[1] +LEARNER = ROOT / "playbook/content/docs/polymarket/learner" +CANONICAL = [ + ROOT / "polymarket/db/schema.sql", + ROOT / "polymarket/db/queries.sql", +] + + +def statements(text: str) -> list[str]: + without_comments = re.sub(r"^\s*--.*$", "", text, flags=re.MULTILINE) + return [ + " ".join(statement.split()) + for statement in without_comments.split(";") + if statement.strip() + ] + + +learner_statements: set[str] = set() +for page in LEARNER.glob("*.mdx"): + text = page.read_text() + for fence in re.findall(r"```sql\s*\n(.*?)```", text, flags=re.DOTALL): + learner_statements.update(statements(fence)) + +if "--print-selects" in sys.argv: + for statement in sorted(learner_statements): + if statement.upper().startswith(("SELECT ", "WITH ")): + print(statement + ";") + sys.exit(0) + +missing = [] +for source in CANONICAL: + for statement in statements(source.read_text()): + if statement not in learner_statements: + missing.append((source.relative_to(ROOT), statement[:120])) + +if missing: + for source, preview in missing: + print(f"ERROR: {source} SQL is missing or different in learner copy blocks: {preview}") + sys.exit(1) + +print("Polymarket canonical SQL matches learner copy blocks.") diff --git a/workshops/build_workshop/scripts/check-windows.ps1 b/workshops/build_workshop/scripts/check-windows.ps1 index 2710bf0..d5ce54d 100644 --- a/workshops/build_workshop/scripts/check-windows.ps1 +++ b/workshops/build_workshop/scripts/check-windows.ps1 @@ -7,6 +7,8 @@ $Setup = Join-Path $ContentRoot 'learner/00-setup.mdx' $Troubleshooting = Join-Path $ContentRoot 'learner/troubleshooting.mdx' $PlatformComponent = Join-Path $WorkshopRoot 'playbook/src/components/platform.tsx' $DocsPage = Join-Path $WorkshopRoot 'playbook/src/app/docs/[[...slug]]/page.tsx' +$PolymarketSetup = Join-Path $ContentRoot 'polymarket/learner/00-setup.mdx' +$PolymarketTroubleshooting = Join-Path $ContentRoot 'polymarket/learner/troubleshooting.mdx' function Assert-Literal { param( @@ -41,6 +43,11 @@ Assert-Literal 'preflight path from repository root' 'workshops/build_workshop/a Assert-Literal 'wrong-shell recovery' 'not recognized" in PowerShell' $Troubleshooting Assert-Literal 'CRLF recovery' "bash\r" $Troubleshooting Assert-Literal 'OAuth browser recovery' 'paste it into the normal Windows' $Troubleshooting +Assert-Literal 'Polymarket requires Ubuntu WSL 2' 'Ubuntu on WSL 2' $PolymarketSetup +Assert-Literal 'Polymarket WSL install command' 'wsl --install -d Ubuntu' $PolymarketSetup +Assert-Literal 'Polymarket Linux-home checkout' 'Keep the repo under `/home`, not `/mnt/c`' $PolymarketSetup +Assert-Literal 'Polymarket environment reload' 'set -a; source ./.env.polymarket; set +a' $PolymarketSetup +Assert-Literal 'Polymarket cross-platform sed backup form' "sed -i.bak" $PolymarketTroubleshooting $shellScripts = @(Get-ChildItem -Path $WorkshopRoot -Recurse -File -Include '*.sh', '*.bash') foreach ($script in $shellScripts) { From 00c85e009bb30f0aa3a51ddeb42a34e067632739 Mon Sep 17 00:00:00 2001 From: Maruthi Prithivi Date: Wed, 29 Jul 2026 17:57:35 +0800 Subject: [PATCH 2/2] docs: document the Polymarket workshop track --- README.md | 1 + workshops/build_workshop/playbook/README.md | 42 ++++++++++++--------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7a5678c..fac014f 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Collection of ClickHouse demo projects showcasing various features and patterns. | ------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | | [Incremental Materialized Views](./incremental_materialized_views/) | Progressive tutorial from basic MVs to full Medallion Architecture | | [Telco Marketing Analytics](./agent_stack_builds/telco_marketing/) | AI-powered telco analytics stack with LibreChat, ClickHouse MCP, LiteLLM, and Langfuse | +| [ClickHouse Cloud Workshops](./workshops/build_workshop/) | AI SRE and Polymarket real-time analytics tracks built on ClickHouse Cloud | ## Getting Started diff --git a/workshops/build_workshop/playbook/README.md b/workshops/build_workshop/playbook/README.md index 892d05f..acf1dda 100644 --- a/workshops/build_workshop/playbook/README.md +++ b/workshops/build_workshop/playbook/README.md @@ -1,13 +1,15 @@ -# ClickHouse BUILD Workshop playbook +# ClickHouse Cloud workshop playbook -The published playbook for the ClickHouse BUILD Workshop ("Build AI with AI"): a -three-hour, hands-on session where participants use their own agentic coding tool to take -an NYC-taxi ride-hailing analytics app end to end on ClickHouse Cloud. +The published playbook offers two dedicated ClickHouse Cloud use cases: -This directory is the documentation site only. The workshop app that participants clone -and build on lives alongside it at `workshops/build_workshop/app` in this repository, on -the `build-workshop-v1` branch. The site is built with [Next.js](https://nextjs.org) -and [Fumadocs](https://fumadocs.dev). Production is +- **AI SRE:** a three-hour NYC-taxi application, observability, incident, and traced-chat track. +- **Polymarket:** a two-hour public market-data stream, real-time aggregate, investigation, and Cloud dashboard track. + +This directory is the documentation site only. The AI SRE application lives at +`workshops/build_workshop/app`; the Polymarket collector and SQL assets live at +`workshops/build_workshop/polymarket`. Learner materials use the `build-workshop-v1` +branch. The site is built with [Next.js](https://nextjs.org) and +[Fumadocs](https://fumadocs.dev). Production is [workshop.demohouse.cloud](https://workshop.demohouse.cloud); dev is [dev-workshop.demohouse.cloud](https://dev-workshop.demohouse.cloud). @@ -62,21 +64,27 @@ the recommended Node deployment redirects those paths to their new module number ## Content authoring -All content is MDX under `content/docs/`. The site is dual-track: every module has a -Learner page and an Instructor page. +All content is MDX under `content/docs/`. Each use case has learner and instructor +tracks with a shared module sequence. ``` content/docs/ - index.mdx # the overview (hero, tracks, scope, modules table, ...) - meta.json # top-level ordering: index, learner, instructor + index.mdx # use-case selector and shared platform contract + ai-sre.mdx # AI SRE landing page; legacy module URLs stay valid + meta.json # top-level ordering and navigation groups learner/ - meta.json # root:true -> Learner track tab; orders the modules - index.mdx # track landing + meta.json # AI SRE learner track; orders the modules + index.mdx # AI SRE learner landing 00-setup.mdx ... 09-wrap-up.mdx instructor/ - meta.json # root:true -> Instructor track tab; orders the modules - index.mdx # run of show + shared-resource checklist + meta.json # AI SRE instructor track; orders the modules + index.mdx # AI SRE run of show + shared-resource checklist 00-setup.mdx ... 09-wrap-up.mdx + polymarket/ + index.mdx # Polymarket use-case landing + rehearsal.mdx # dev-only maintainer rehearsal + learner/ # 00-setup ... 07-wrap-up + troubleshooting + instructor/ # matching 00-setup ... 07-wrap-up run of show ``` - Ordering is controlled by the `pages` array in each `meta.json` (file basenames, @@ -104,7 +112,7 @@ Desktop WSL integration; it does not mean translating shared Bash blocks into Po - Keep shell scripts LF-only. The Windows CI job enforces the platform contract and builds the complete playbook on a Windows runner. -### The per-module learner contract +### The AI SRE per-module learner contract Every learner module page follows this skeleton, in order: