From 4e3c05a124a4a5e318c60ed6c766d90dc2b54cc7 Mon Sep 17 00:00:00 2001 From: sishuo Date: Sun, 2 Aug 2026 16:48:23 +0800 Subject: [PATCH 1/2] Add workshop_public: snowflake migration lab and RTA-mini lab artifacts --- workshop_public/RTA-mini-workshop/.gitignore | 21 + workshop_public/RTA-mini-workshop/README.md | 40 ++ .../RTA-mini-workshop/dashboard/.dockerignore | 7 + .../RTA-mini-workshop/dashboard/.env.example | 20 + .../RTA-mini-workshop/dashboard/Dockerfile | 16 + .../RTA-mini-workshop/dashboard/README.md | 69 +++ .../RTA-mini-workshop/dashboard/app.py | 198 +++++++ .../dashboard/docker-compose.yml | 8 + .../dashboard/requirements.txt | 4 + .../RTA-mini-workshop/dashboard/static/app.js | 248 +++++++++ .../static/clickhouse-logo-black.png | Bin 0 -> 26361 bytes .../dashboard/static/index.html | 83 +++ .../dashboard/static/styles.css | 102 ++++ .../RTA-mini-workshop/dashboard/test_app.py | 78 +++ .../snowflake_migration_lab/.gitignore | 65 +++ .../01-setup-snowflake/.env.example | 34 ++ .../01-setup-snowflake/.gitignore | 1 + .../dbt/nyc_taxi_dbt/dbt_project.yml | 31 ++ .../macros/generate_schema_name.sql | 10 + .../macros/generate_surrogate_key.sql | 5 + .../analytics/agg_hourly_zone_trips.sql | 31 ++ .../models/analytics/dim_date.sql | 90 +++ .../models/analytics/dim_payment_type.sql | 10 + .../models/analytics/dim_taxi_zones.sql | 15 + .../models/analytics/dim_vendor.sql | 9 + .../models/analytics/fact_trips.sql | 46 ++ .../nyc_taxi_dbt/models/analytics/schema.yml | 39 ++ .../intermediate/int_trips_enriched.sql | 72 +++ .../dbt/nyc_taxi_dbt/models/sources.yml | 23 + .../nyc_taxi_dbt/models/staging/schema.yml | 44 ++ .../models/staging/stg_taxi_zones.sql | 16 + .../nyc_taxi_dbt/models/staging/stg_trips.sql | 51 ++ .../dbt/nyc_taxi_dbt/package-lock.yml | 11 + .../dbt/nyc_taxi_dbt/packages.yml | 5 + .../dbt/nyc_taxi_dbt/profiles.yml.example | 29 + .../snapshots/driver_rating_snapshot.sql | 25 + .../tests/assert_revenue_positive.sql | 9 + .../tests/assert_trips_not_future.sql | 8 + .../01-setup-snowflake/producer/Dockerfile | 14 + .../01-setup-snowflake/producer/producer.py | 281 ++++++++++ .../producer/requirements.txt | 2 + .../queries/q1_hourly_revenue_by_borough.sql | 35 ++ .../queries/q2_rolling_7day_avg_distance.sql | 39 ++ .../queries/q3_top10_trips_qualify.sql | 38 ++ .../q4_driver_rating_lateral_flatten.sql | 37 ++ .../queries/q5_surge_pricing_variant.sql | 39 ++ .../queries/q6_merge_hourly_aggregation.sql | 63 +++ .../queries/q7_cdc_stream_consumption.sql | 43 ++ .../scripts/01_create_tables.sql | 115 ++++ .../scripts/02_seed_data.sql | 207 +++++++ .../scripts/04_create_secure_view.sql | 32 ++ .../01-setup-snowflake/scripts/run_dbt.sh | 151 +++++ .../scripts/validate_environment.sql | 77 +++ .../scripts/verify_environment.sh | 233 ++++++++ .../01-setup-snowflake/setup.sh | 524 ++++++++++++++++++ .../01-setup-snowflake/superset/Dockerfile | 13 + .../01_operations_command_center.zip | Bin 0 -> 15388 bytes .../dashboards/02_executive_weekly_report.zip | Bin 0 -> 17404 bytes .../03_driver_quality_analytics.zip | Bin 0 -> 17996 bytes .../superset/docker-compose.yml | 110 ++++ .../superset/init_superset.sh | 213 +++++++ .../superset/superset_config.py | 46 ++ .../01-setup-snowflake/teardown.sh | 93 ++++ .../terraform/backend.hcl.example | 12 + .../01-setup-snowflake/terraform/database.tf | 64 +++ .../01-setup-snowflake/terraform/main.tf | 40 ++ .../01-setup-snowflake/terraform/monitors.tf | 14 + .../01-setup-snowflake/terraform/outputs.tf | 44 ++ .../01-setup-snowflake/terraform/roles.tf | 403 ++++++++++++++ .../01-setup-snowflake/terraform/streams.tf | 103 ++++ .../terraform/terraform.tfvars.example | 13 + .../01-setup-snowflake/terraform/variables.tf | 40 ++ .../terraform/warehouses.tf | 16 + .../02-plan-and-design/migration-plan.md | 231 ++++++++ .../scripts/01_profile_snowflake.sh | 424 ++++++++++++++ .../scripts/02_query_history.sql | 128 +++++ .../01_mergetree_engine_selection.md | 120 ++++ .../worksheets/02_sort_key_design.md | 157 ++++++ .../worksheets/03_schema_translation.md | 204 +++++++ .../worksheets/04_migration_wave_plan.md | 183 ++++++ .../worksheets/05_dbt_model_design.md | 191 +++++++ .../03-migrate-to-clickhouse/.env.example | 31 ++ .../03-migrate-to-clickhouse/.gitignore | 8 + .../dbt/nyc_taxi_dbt_ch/dbt_project.yml | 60 ++ .../macros/generate_schema_name.sql | 19 + .../intermediate/int_trips_enriched.sql | 95 ++++ .../models/marts/agg_hourly_zone_trips.sql | 70 +++ .../nyc_taxi_dbt_ch/models/marts/dim_date.sql | 153 +++++ .../models/marts/dim_payment_type.sql | 36 ++ .../models/marts/dim_taxi_zones.sql | 29 + .../models/marts/dim_vendor.sql | 31 ++ .../models/marts/fact_trips.sql | 111 ++++ .../nyc_taxi_dbt_ch/models/marts/schema.yml | 145 +++++ .../materialized_views/mv_live_trip_feed.sql | 38 ++ .../dbt/nyc_taxi_dbt_ch/models/sources.yml | 66 +++ .../nyc_taxi_dbt_ch/models/staging/schema.yml | 99 ++++ .../models/staging/stg_taxi_zones.sql | 36 ++ .../models/staging/stg_trips.sql | 106 ++++ .../dbt/nyc_taxi_dbt_ch/package-lock.yml | 5 + .../dbt/nyc_taxi_dbt_ch/packages.yml | 11 + .../dbt/nyc_taxi_dbt_ch/profiles.yml.example | 27 + .../tests/assert_revenue_positive.sql | 23 + .../tests/assert_trips_not_future.sql | 18 + .../producer/Dockerfile | 10 + .../producer/producer.py | 239 ++++++++ .../producer/requirements.txt | 1 + .../scripts/00_seed_zones.sql | 65 +++ .../scripts/01_verify_migration.sh | 195 +++++++ .../scripts/02_migrate_trips.py | 298 ++++++++++ .../scripts/02_validate_parity.sql | 50 ++ .../scripts/03_cutover.sh | 235 ++++++++ .../scripts/04_create_dictionary.sql | 26 + .../scripts/run_benchmark.sh | 342 ++++++++++++ .../03-migrate-to-clickhouse/setup.sh | 184 ++++++ .../superset/add_clickhouse_connection.sh | 206 +++++++ .../dashboard_export_20260401T084808.zip | Bin 0 -> 23747 bytes .../03-migrate-to-clickhouse/teardown.sh | 86 +++ .../terraform/main.tf | 15 + .../terraform/outputs.tf | 19 + .../terraform/service.tf | 21 + .../terraform/variables.tf | 40 ++ .../04-evaluation/assessment.md | 393 +++++++++++++ .../snowflake_migration_lab/README.md | 38 ++ 123 files changed, 10045 insertions(+) create mode 100644 workshop_public/RTA-mini-workshop/.gitignore create mode 100644 workshop_public/RTA-mini-workshop/README.md create mode 100644 workshop_public/RTA-mini-workshop/dashboard/.dockerignore create mode 100644 workshop_public/RTA-mini-workshop/dashboard/.env.example create mode 100644 workshop_public/RTA-mini-workshop/dashboard/Dockerfile create mode 100644 workshop_public/RTA-mini-workshop/dashboard/README.md create mode 100644 workshop_public/RTA-mini-workshop/dashboard/app.py create mode 100644 workshop_public/RTA-mini-workshop/dashboard/docker-compose.yml create mode 100644 workshop_public/RTA-mini-workshop/dashboard/requirements.txt create mode 100644 workshop_public/RTA-mini-workshop/dashboard/static/app.js create mode 100644 workshop_public/RTA-mini-workshop/dashboard/static/clickhouse-logo-black.png create mode 100644 workshop_public/RTA-mini-workshop/dashboard/static/index.html create mode 100644 workshop_public/RTA-mini-workshop/dashboard/static/styles.css create mode 100644 workshop_public/RTA-mini-workshop/dashboard/test_app.py create mode 100644 workshop_public/snowflake_migration_lab/.gitignore create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/.env.example create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/.gitignore create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/dbt_project.yml create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_schema_name.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_surrogate_key.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/agg_hourly_zone_trips.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_date.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_payment_type.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_taxi_zones.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_vendor.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/fact_trips.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/schema.yml create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/intermediate/int_trips_enriched.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/sources.yml create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/schema.yml create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_taxi_zones.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_trips.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/package-lock.yml create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/packages.yml create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/profiles.yml.example create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/snapshots/driver_rating_snapshot.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_revenue_positive.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_trips_not_future.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/Dockerfile create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/producer.py create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/requirements.txt create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q1_hourly_revenue_by_borough.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q2_rolling_7day_avg_distance.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q3_top10_trips_qualify.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q4_driver_rating_lateral_flatten.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q5_surge_pricing_variant.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q6_merge_hourly_aggregation.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q7_cdc_stream_consumption.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/01_create_tables.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/02_seed_data.sql create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/04_create_secure_view.sql create mode 100755 workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/run_dbt.sh create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/validate_environment.sql create mode 100755 workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/verify_environment.sh create mode 100755 workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/Dockerfile create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/01_operations_command_center.zip create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/02_executive_weekly_report.zip create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/03_driver_quality_analytics.zip create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/docker-compose.yml create mode 100755 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/init_superset.sh create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/superset_config.py create mode 100755 workshop_public/snowflake_migration_lab/01-setup-snowflake/teardown.sh create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/backend.hcl.example create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/database.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/main.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/monitors.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/outputs.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/roles.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/streams.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/terraform.tfvars.example create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/variables.tf create mode 100644 workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/warehouses.tf create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/migration-plan.md create mode 100755 workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/02_query_history.sql create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md create mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.env.example create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.gitignore create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/dbt_project.yml create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/macros/generate_schema_name.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/intermediate/int_trips_enriched.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/agg_hourly_zone_trips.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_date.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_payment_type.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_taxi_zones.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_vendor.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/fact_trips.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/schema.yml create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/materialized_views/mv_live_trip_feed.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/sources.yml create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/schema.yml create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_taxi_zones.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_trips.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/package-lock.yml create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/packages.yml create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/profiles.yml.example create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_revenue_positive.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_trips_not_future.sql create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/Dockerfile create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/producer.py create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/requirements.txt create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/00_seed_zones.sql create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/01_verify_migration.sh create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_migrate_trips.py create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_validate_parity.sql create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/03_cutover.sh create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/04_create_dictionary.sql create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/run_benchmark.sh create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/superset/add_clickhouse_connection.sh create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/superset/dashboards/dashboard_export_20260401T084808.zip create mode 100755 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/teardown.sh create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/main.tf create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/outputs.tf create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/service.tf create mode 100644 workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/variables.tf create mode 100644 workshop_public/snowflake_migration_lab/04-evaluation/assessment.md create mode 100644 workshop_public/snowflake_migration_lab/README.md diff --git a/workshop_public/RTA-mini-workshop/.gitignore b/workshop_public/RTA-mini-workshop/.gitignore new file mode 100644 index 0000000..5b00480 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/.gitignore @@ -0,0 +1,21 @@ +# Secrets — never commit real connection details +.env +dashboard/.env +**/.env + +# Python +__pycache__/ +*.pyc +*.pyo +.venv/ +venv/ +env/ + +# Pages build output +_site/ + +# OS / editor cruft +.DS_Store +Thumbs.db +.idea/ +.vscode/ diff --git a/workshop_public/RTA-mini-workshop/README.md b/workshop_public/RTA-mini-workshop/README.md new file mode 100644 index 0000000..982b120 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/README.md @@ -0,0 +1,40 @@ +# RTA Mini Workshop — Real-Time Market Analytics with ClickHouse + +A ~90-minute, self-contained hands-on lab: participants sign up for ClickHouse +Cloud, load ~26.5M forex ticks from public object storage, run real +market-analytics queries, try the built-in AI Assistant and Agents, and +(optionally) run a live dashboard on their own data. Generic and partner-neutral. + +## Where the pieces live + +The workshops are served by the workshop site, so the **guide** is served from +there and only the **dashboard** (a separate service) lives here: + +| Piece | Location | Served at | +|---|---|---| +| Participant guide (MDX docs tree) | `site/content/docs/rta-mini/` | `https://workshop.demohouse.cloud/docs/rta-mini` | +| Optional live dashboard | `dashboard/` (this folder) | run locally with Docker — see `dashboard/README.md` | + +The guide is linked from the workshops hub (the app's `/` route). + +## The workshop data + +The guide loads a public forex Parquet dataset via a ClickPipe and via a one-shot +`s3(...)` query: + +``` +https://partner-workshop.s3.ap-southeast-1.amazonaws.com/fx/ticks.parquet +``` + +> **Note.** The dataset lives in a neutral, public S3 bucket (`partner-workshop` +> in `ap-southeast-1`), so the lab is fully partner-neutral and works with both +> the ClickPipes S3 source and the `s3()` function (anonymous `GetObject` + +> `ListBucket`). To host your own copy, upload `fx/ticks.parquet` to a public +> bucket and update the two references in the guide +> (`site/content/docs/rta-mini/learner/load-data.mdx`, served at +> `https://workshop.demohouse.cloud/docs/rta-mini`). + +## Prerequisites (participants) + +- A ClickHouse Cloud account (free trial — created during the lab). +- For the optional dashboard: Docker Desktop. diff --git a/workshop_public/RTA-mini-workshop/dashboard/.dockerignore b/workshop_public/RTA-mini-workshop/dashboard/.dockerignore new file mode 100644 index 0000000..051a4d0 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/.dockerignore @@ -0,0 +1,7 @@ +.env +__pycache__/ +*.pyc +.venv/ +venv/ +.git/ +.DS_Store diff --git a/workshop_public/RTA-mini-workshop/dashboard/.env.example b/workshop_public/RTA-mini-workshop/dashboard/.env.example new file mode 100644 index 0000000..86b651f --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/.env.example @@ -0,0 +1,20 @@ +# Copy this file to ".env" and fill in your ClickHouse Cloud connection details. +# Find them in the Cloud console: left menu -> Connect -> "HTTPS" / "Native". + +# Host looks like: abc123xyz.ap-southeast-1.aws.clickhouse.cloud (no https://, no port) +CLICKHOUSE_HOST=your-service-id.ap-southeast-1.aws.clickhouse.cloud + +# HTTPS port for ClickHouse Cloud is 8443. +CLICKHOUSE_PORT=8443 + +# Default user unless you created another one. +CLICKHOUSE_USER=default + +# The password you set when you created the service. +CLICKHOUSE_PASSWORD=your-service-password + +# Database that holds the `forex` table (usually "default"). +CLICKHOUSE_DATABASE=default + +# ClickHouse Cloud always uses TLS. +CLICKHOUSE_SECURE=true diff --git a/workshop_public/RTA-mini-workshop/dashboard/Dockerfile b/workshop_public/RTA-mini-workshop/dashboard/Dockerfile new file mode 100644 index 0000000..e92d8f6 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/Dockerfile @@ -0,0 +1,16 @@ +# Pin the Python version here so it doesn't matter what's on your laptop. +FROM python:3.12-slim + +WORKDIR /app + +# Install dependencies first for better layer caching. +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# App code + static front end. +COPY app.py . +COPY static ./static + +EXPOSE 8000 + +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/workshop_public/RTA-mini-workshop/dashboard/README.md b/workshop_public/RTA-mini-workshop/dashboard/README.md new file mode 100644 index 0000000..8a25c09 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/README.md @@ -0,0 +1,69 @@ +# Forex Live Dashboard + +A small dashboard that visualizes the `forex` table you loaded in the workshop, +served from your own ClickHouse Cloud service. Change a filter and watch the +query time — that's the point: ClickHouse answers in milliseconds. + +- **Backend:** FastAPI + `clickhouse-connect` +- **Charts:** Apache ECharts (candlestick + volume, interactive zoom) +- **Runs in Docker** so your laptop's Python version doesn't matter. + +## Prerequisites + +- [Docker Desktop](https://www.docker.com/products/docker-desktop/) (includes `docker compose`). +- A running ClickHouse Cloud service with the `forex` table loaded (Steps 1–3 of the workshop). + +## Setup + +1. Clone the repo and go into this folder: + + ```bash + git clone https://github.com/ClickHouse/ClickHouse_Demos.git + cd ClickHouse_Demos + cd "$(git rev-parse --show-toplevel)/workshop_public/RTA-mini-workshop/dashboard" + ``` + +2. Create your `.env` from the template and fill in your connection details: + + ```bash + cp .env.example .env + ``` + + Find the values in the Cloud console under **Connect → HTTPS/Native**: + + | Variable | Value | + | --- | --- | + | `CLICKHOUSE_HOST` | e.g. `abc123.ap-southeast-1.aws.clickhouse.cloud` (no `https://`, no port) | + | `CLICKHOUSE_PORT` | `8443` | + | `CLICKHOUSE_USER` | `default` | + | `CLICKHOUSE_PASSWORD` | the password you set when creating the service | + | `CLICKHOUSE_DATABASE` | `default` | + | `CLICKHOUSE_SECURE` | `true` | + +3. Build and run: + + ```bash + docker compose up --build + ``` + +4. Open . + +Stop it with `Ctrl-C`, or `docker compose down` from another terminal. + +## Run without Docker (optional) + +Needs Python 3.9+: + +```bash +pip install -r requirements.txt +cp .env.example .env # then edit it +uvicorn app:app --reload --port 8000 +``` + +## Troubleshooting + +- **Red banner "Could not reach ClickHouse":** re-check the values in `.env`. The + host must have no `https://` prefix and no port suffix; the port is `8443`. +- **"forex table not found":** load the data first (workshop Steps 2–3). +- **Port 8000 in use:** change the mapping in `docker-compose.yml`, e.g. + `"8080:8000"`, then open . diff --git a/workshop_public/RTA-mini-workshop/dashboard/app.py b/workshop_public/RTA-mini-workshop/dashboard/app.py new file mode 100644 index 0000000..b2fb757 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/app.py @@ -0,0 +1,198 @@ +""" +ClickHouse Forex Live Dashboard — FastAPI backend. + +Serves a small single-page dashboard (static/) and a JSON API that queries the +`forex` table you loaded during the workshop. Every API response carries the +ClickHouse query timing and rows-scanned count, so the front end can show how +little work the database does when a filter hits the sort key. + +Runs on Python 3.9+ (the Docker image pins 3.12, so your laptop's Python +version doesn't matter). +""" + +import os +import time +from datetime import datetime, timedelta + +from fastapi import FastAPI, HTTPException, Query +from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles + +# Load a local .env if present (Docker passes these via env_file instead). +try: + from dotenv import load_dotenv + + load_dotenv() +except Exception: # python-dotenv is optional at runtime + pass + +import clickhouse_connect + +app = FastAPI(title="ClickHouse Forex Live Dashboard") + +_client = None + + +def get_client(): + """Create (once) and reuse a ClickHouse Cloud client from env vars.""" + global _client + if _client is None: + host = os.getenv("CLICKHOUSE_HOST") + if not host: + raise HTTPException( + status_code=500, + detail="CLICKHOUSE_HOST is not set. Copy .env.example to .env " + "and fill in your ClickHouse Cloud connection details.", + ) + _client = clickhouse_connect.get_client( + host=host, + port=int(os.getenv("CLICKHOUSE_PORT", "8443")), + username=os.getenv("CLICKHOUSE_USER", "default"), + password=os.getenv("CLICKHOUSE_PASSWORD", ""), + database=os.getenv("CLICKHOUSE_DATABASE", "default"), + secure=os.getenv("CLICKHOUSE_SECURE", "true").lower() in ("1", "true", "yes"), + connect_timeout=15, + query_limit=0, + ) + return _client + + +def _run(sql, params=None): + """Run a query and return (result, wall_ms, rows_read, server_ms).""" + client = get_client() + t0 = time.perf_counter() + res = client.query(sql, parameters=params or {}) + wall_ms = (time.perf_counter() - t0) * 1000.0 + summary = res.summary or {} + rows_read = int(summary.get("read_rows", 0) or 0) + elapsed_ns = summary.get("elapsed_ns") + server_ms = (int(elapsed_ns) / 1_000_000.0) if elapsed_ns else None + return res, wall_ms, rows_read, server_ms + + +@app.get("/api/health") +def health(): + try: + _run("SELECT 1") + return {"ok": True} + except Exception as exc: # surface config/connection problems to the UI + return JSONResponse(status_code=500, content={"ok": False, "error": str(exc)}) + + +@app.get("/api/meta") +def meta(): + """Currency pairs and the available date range, for the filter controls.""" + res, _, _, _ = _run( + "SELECT DISTINCT concat(base, '/', quote) AS pair FROM forex ORDER BY pair" + ) + pairs = [row[0] for row in res.result_rows] + res2, _, _, _ = _run( + "SELECT toDate(min(datetime)) AS a, toDate(max(datetime)) AS b FROM forex" + ) + dmin, dmax = res2.result_rows[0] + return {"pairs": pairs, "date_min": str(dmin), "date_max": str(dmax)} + + +@app.get("/api/dashboard") +def dashboard( + pair: str = Query(..., description="e.g. XAU/USD"), + start: str = Query(..., description="YYYY-MM-DD (inclusive)"), + end: str = Query(..., description="YYYY-MM-DD (inclusive)"), + bucket: str = Query("day", pattern="^(day|hour)$"), +): + """OHLC candles + volume + KPIs for one pair over a date range.""" + if "/" not in pair: + raise HTTPException(400, "pair must look like BASE/QUOTE, e.g. XAU/USD") + base, quote = pair.split("/", 1) + + try: + start_date = datetime.strptime(start, "%Y-%m-%d") + end_date = datetime.strptime(end, "%Y-%m-%d") + if start_date > end_date: + raise HTTPException(400, "start must be on or before end") + start_dt = start_date.strftime("%Y-%m-%d %H:%M:%S") + # end is inclusive, so we scan up to the start of the following day + end_dt = (end_date + timedelta(days=1)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + except ValueError: + raise HTTPException(400, "start and end must be YYYY-MM-DD") + + bucket_expr = "toStartOfHour(datetime)" if bucket == "hour" else "toStartOfDay(datetime)" + params = {"base": base, "quote": quote, "start": start_dt, "end": end_dt} + where = ( + "base = {base:String} AND quote = {quote:String} " + "AND datetime >= {start:DateTime} AND datetime < {end:DateTime}" + ) + + ohlc_sql = f""" + SELECT {bucket_expr} AS t, + argMin(bid, datetime) AS open, + max(bid) AS high, + min(bid) AS low, + argMax(bid, datetime) AS close, + count() AS volume, + round(avg(ask - bid), 6) AS avg_spread + FROM forex + WHERE {where} + GROUP BY t + ORDER BY t + """ + kpi_sql = f""" + SELECT count() AS ticks, + round(quantile(0.5)(ask - bid), 6) AS median_spread, + round(quantile(0.99)(ask - bid), 6) AS p99_spread, + argMax(bid, datetime) AS last_bid, + min(bid) AS low, + max(bid) AS high + FROM forex + WHERE {where} + """ + + r1, w1, rr1, s1 = _run(ohlc_sql, params) + r2, w2, rr2, s2 = _run(kpi_sql, params) + + ohlc = [ + { + "t": str(row[0]), + "open": row[1], + "high": row[2], + "low": row[3], + "close": row[4], + "volume": int(row[5]), + "spread": row[6], + } + for row in r1.result_rows + ] + + if r2.result_rows and r2.result_rows[0][0]: + k = r2.result_rows[0] + kpis = { + "ticks": int(k[0]), + "median_spread": k[1], + "p99_spread": k[2], + "last_bid": k[3], + "low": k[4], + "high": k[5], + } + else: + kpis = {"ticks": 0, "median_spread": None, "p99_spread": None, + "last_bid": None, "low": None, "high": None} + + server_ms = round(s1 + s2, 2) if (s1 is not None and s2 is not None) else None + timing = { + "server_ms": server_ms, + "wall_ms": round(w1 + w2, 2), + "rows_read": rr1 + rr2, + "queries": 2, + } + return {"ohlc": ohlc, "kpis": kpis, "timing": timing, "bucket": bucket, "pair": pair} + + +# Static single-page front end. Mount last so /api/* routes win. +app.mount("/static", StaticFiles(directory="static"), name="static") + + +@app.get("/") +def index(): + return FileResponse("static/index.html") diff --git a/workshop_public/RTA-mini-workshop/dashboard/docker-compose.yml b/workshop_public/RTA-mini-workshop/dashboard/docker-compose.yml new file mode 100644 index 0000000..099cf24 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/docker-compose.yml @@ -0,0 +1,8 @@ +services: + dashboard: + build: . + ports: + - "8000:8000" + env_file: + - .env + restart: unless-stopped diff --git a/workshop_public/RTA-mini-workshop/dashboard/requirements.txt b/workshop_public/RTA-mini-workshop/dashboard/requirements.txt new file mode 100644 index 0000000..c27e6e1 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/requirements.txt @@ -0,0 +1,4 @@ +fastapi>=0.111,<1.0 +uvicorn[standard]>=0.30,<1.0 +clickhouse-connect>=0.8.0,<1.0 +python-dotenv>=1.0,<2.0 diff --git a/workshop_public/RTA-mini-workshop/dashboard/static/app.js b/workshop_public/RTA-mini-workshop/dashboard/static/app.js new file mode 100644 index 0000000..7ad6913 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/static/app.js @@ -0,0 +1,248 @@ +/* ClickHouse Forex Live Dashboard — front end. + Fetches /api/dashboard on every filter change and renders an ECharts + candlestick + volume view, plus a prominent query-latency badge. */ + +const UP = "#2F7D4F"; +const DOWN = "#C0392B"; +const ACCENT = "#FFE000"; + +const el = (id) => document.getElementById(id); +const chart = echarts.init(el("chart")); +window.addEventListener("resize", () => chart.resize()); + +let meta = { pairs: [], date_min: null, date_max: null }; +let bucket = "day"; + +function showBanner(msg) { + const b = el("banner"); + b.textContent = msg; + b.classList.add("show"); +} +function hideBanner() { + el("banner").classList.remove("show"); +} + +function addDays(iso, n) { + const d = new Date(iso + "T00:00:00Z"); + d.setUTCDate(d.getUTCDate() + n); + return d.toISOString().slice(0, 10); +} + +function fmtPrice(v) { + if (v == null) return "—"; + const abs = Math.abs(v); + const digits = abs >= 100 ? 2 : abs >= 1 ? 4 : 6; + return Number(v).toLocaleString(undefined, { minimumFractionDigits: digits, maximumFractionDigits: digits }); +} +function fmtInt(v) { + if (v == null) return "—"; + return Number(v).toLocaleString(); +} + +async function init() { + try { + const res = await fetch("/api/meta"); + if (!res.ok) throw new Error((await res.json()).detail || res.statusText); + meta = await res.json(); + } catch (e) { + showBanner( + "Could not reach ClickHouse. Check your .env connection details and that the forex table is loaded. Details: " + + e.message + ); + return; + } + + const pairSel = el("pair"); + meta.pairs.forEach((p) => { + const o = document.createElement("option"); + o.value = p; + o.textContent = p; + pairSel.appendChild(o); + }); + // Default to gold if present — it's the liveliest series in the dataset. + pairSel.value = meta.pairs.includes("XAU/USD") ? "XAU/USD" : meta.pairs[0]; + + el("start").min = el("end").min = meta.date_min; + el("start").max = el("end").max = meta.date_max; + el("start").value = meta.date_min; + el("end").value = meta.date_max; + + // Wire up controls. + pairSel.addEventListener("change", load); + el("start").addEventListener("change", load); + el("end").addEventListener("change", load); + + el("bucket").querySelectorAll("button").forEach((btn) => { + btn.addEventListener("click", () => { + bucket = btn.dataset.bucket; + el("bucket").querySelectorAll("button").forEach((b) => b.classList.toggle("on", b === btn)); + load(); + }); + }); + + el("presets").querySelectorAll("button").forEach((btn) => { + btn.addEventListener("click", () => applyPreset(btn.dataset.preset)); + }); + + load(); +} + +function applyPreset(kind) { + if (kind === "month") { + el("start").value = meta.date_min; + el("end").value = meta.date_max; + setBucket("day"); + } else if (kind === "week") { + el("start").value = meta.date_min; + el("end").value = addDays(meta.date_min, 6); + setBucket("hour"); + } else if (kind === "day") { + // A day mid-dataset so there's plenty of data on either side. + const d = addDays(meta.date_min, 6); + el("start").value = d; + el("end").value = d; + setBucket("hour"); + } + load(); +} + +function setBucket(b) { + bucket = b; + el("bucket").querySelectorAll("button").forEach((btn) => btn.classList.toggle("on", btn.dataset.bucket === b)); +} + +async function load() { + const pair = el("pair").value; + const start = el("start").value; + const end = el("end").value; + if (!pair || !start || !end) return; + if (start > end) { + showBanner("The From date is after the To date."); + return; + } + hideBanner(); + chart.showLoading("default", { text: "", color: ACCENT, maskColor: "rgba(251,251,249,0.6)" }); + + const qs = new URLSearchParams({ pair, start, end, bucket }); + let data; + try { + const res = await fetch("/api/dashboard?" + qs.toString()); + if (!res.ok) throw new Error((await res.json()).detail || res.statusText); + data = await res.json(); + } catch (e) { + chart.hideLoading(); + showBanner("Query failed: " + e.message); + return; + } + chart.hideLoading(); + + updateLatency(data.timing); + updateKpis(data.kpis); + renderChart(data); +} + +function updateLatency(t) { + const ms = t.server_ms != null ? t.server_ms : t.wall_ms; + el("lat-ms").textContent = ms != null ? ms.toFixed(ms < 10 ? 1 : 0) : "—"; + el("lat-label").textContent = t.server_ms != null ? "ClickHouse query time" : "round-trip time"; + el("lat-rows").textContent = fmtInt(t.rows_read); + const badge = el("latency"); + badge.classList.remove("flash"); + void badge.offsetWidth; // restart the animation + badge.classList.add("flash"); +} + +function updateKpis(k) { + el("kpi-ticks").textContent = fmtInt(k.ticks); + el("kpi-last").textContent = fmtPrice(k.last_bid); + el("kpi-range").textContent = + k.low == null ? "—" : fmtPrice(k.low) + " / " + fmtPrice(k.high); + el("kpi-median").textContent = fmtPrice(k.median_spread); + el("kpi-p99").textContent = fmtPrice(k.p99_spread); +} + +function renderChart(data) { + el("chart-title").textContent = data.pair + " — price & volume (" + data.bucket + ")"; + const rows = data.ohlc; + const cats = rows.map((d) => d.t); + const candle = rows.map((d) => [d.open, d.close, d.low, d.high]); + const volume = rows.map((d) => ({ + value: d.volume, + itemStyle: { color: d.close >= d.open ? UP : DOWN }, + })); + + chart.setOption( + { + animationDuration: 250, + textStyle: { fontFamily: "Inter, sans-serif" }, + tooltip: { + trigger: "axis", + axisPointer: { type: "cross" }, + backgroundColor: "#1A1A17", + borderColor: "#1A1A17", + textStyle: { color: "#F2F1EA", fontFamily: "JetBrains Mono, monospace", fontSize: 12 }, + }, + axisPointer: { link: [{ xAxisIndex: "all" }] }, + grid: [ + { left: 62, right: 24, top: 24, height: "58%" }, + { left: 62, right: 24, top: "72%", height: "16%" }, + ], + xAxis: [ + { + type: "category", + data: cats, + boundaryGap: true, + axisLine: { lineStyle: { color: "#B9B8AC" } }, + axisLabel: { color: "#6B6B63", fontSize: 11 }, + splitLine: { show: false }, + }, + { + type: "category", + gridIndex: 1, + data: cats, + axisLine: { lineStyle: { color: "#B9B8AC" } }, + axisLabel: { show: false }, + axisTick: { show: false }, + }, + ], + yAxis: [ + { + scale: true, + splitLine: { lineStyle: { color: "#EEEDE4" } }, + axisLabel: { color: "#6B6B63", fontSize: 11 }, + }, + { + gridIndex: 1, + splitNumber: 2, + axisLabel: { color: "#6B6B63", fontSize: 10 }, + splitLine: { show: false }, + }, + ], + dataZoom: [ + { type: "inside", xAxisIndex: [0, 1] }, + { type: "slider", xAxisIndex: [0, 1], height: 18, bottom: 6, borderColor: "#E6E5DD" }, + ], + series: [ + { + name: "OHLC", + type: "candlestick", + data: candle, + itemStyle: { + color: UP, color0: DOWN, + borderColor: UP, borderColor0: DOWN, + }, + }, + { + name: "Volume", + type: "bar", + xAxisIndex: 1, + yAxisIndex: 1, + data: volume, + }, + ], + }, + true + ); +} + +init(); diff --git a/workshop_public/RTA-mini-workshop/dashboard/static/clickhouse-logo-black.png b/workshop_public/RTA-mini-workshop/dashboard/static/clickhouse-logo-black.png new file mode 100644 index 0000000000000000000000000000000000000000..9b41d54f5d67e176ba75f6e959723db094fea017 GIT binary patch literal 26361 zcmeEuWmHsc^zR`A1CcQRrF2jfX+cU#0Z|kXQ0bOXI);w1V3c~57EnS!q@|>X5EPJ* z8bLxD2SmEz?t|}p{~zv``}wYW)^Z6m&w0*%cKmi8p50PaI=KJDegr`dUcaWGfgt<* z5riUpFAaRcUoN)+|Lt?QrsE91aiaey+*4#-5#%ItUE%U=_qgf)pEymWGSQ)ldt+4+ z$E0MA4QWzH9vdpA|*HwRf?mca=--v)wbgu%M{$<7%C`LfplW zh42-Oqk*zc0sRMo7?0yk6WdMo>aR7h!>QpHtasN+MRIH;%C2x=hxsEpb`(c=;Sjr* z+Ir5;=7j8vm=3~f1umM;=UmE7-2nG&-Tko*e{NhSvtvy)luPW{MoTU%h++o{TJlod zQo`?(8CoM^nDV@TA1Yy3CRx2knSx%#k6X8zj#os~V6leg<0@nZF9#+c_$^ZLwFh4z zE<^PTIqN7R#_>*5Tt%#+QA#esd_afH@b_i}QCmBO&DINW_kW$A6VlGI-X1^b#Mm^l zO1t zmm~_S=+e5U26j{Yx4v>BpqpZOs2qoscJ&E?X#ZZhFNBWP{`U)_oy?4id-oE&@Z99T zpApXg{{`^k|FdLQA^x9Hh6OV+9EdpexjB&F-Y(hT=6ycpM=?ExAT^u!SHo}(#w>aC zz{FeJ{$NK9E-aWF-GJ4QLl~j^Y?Q_){kJb}L=is1&*&=uM)jBI|GU@!C5vQ>=wUhW z&%NA&#=4U&eoQ)LNf}QI@6q-yDR@l$rkKB%<YA5VAS3%MT^?uKZQD-jJ=q-+2%OW z^@Uwb8oIa3)Sz3^lJmA?)~?yczi8v2zsnS&yZN|>LY(aS&o9D$eQU`(s5n|TrA=-> z*heNQDankjGzV!kX5#*WsQ4a~4>q=0e+oUz353A)4kkyE8F^cXf?HT?(1$?SSl?3g z@qRD6oF4A9v%2#%Fa6D%;w)$HT3d^O4Sj0#NcdV!Ix1u+_JZ?aqei2Q*@O zuHiFucP2Lu%X=ZCvAvHl~cLb=T((x=W3JF8FgN1n+m zi>wys4(p~U61mT|yBev#3CB%vZZv)zo)^cpoXRe}CKK6F_xt9N5jrk7KI*Uovmt+9io0T)Wx{q|l;2v^{K&tNTD_AxZiu#s93o z{=?~UE9s)8BpjY?=@rW^!8EhrKM`cLs5#DJ+(%*axUz}a$_1=%C}AfmCSRB4iOQ0s zP-f4M-Nl0LT)=+1TNbLE!(7-8wueuYJSSu{^X7T8M`QnbSmNEApe@Y*NS?C6G<{K5 zHo3?9=0IR``Pb%!`o9Q(Og=%`kDPll_avX-4pt;#@mE#wDyVct!egTLo7ha9$8I+_ zmpu9_1!OP@J&&A);k|Jw?DpQLg02gw>5{uI|K&ULYYshH1Q9c(i`RfZMzs6m9paMWX-*;}{S2rYoe3qU+3wI-o7}D4Y0n zVNcBDvnjd-|*a!et8wF4jlX0A=Lfu1o7R2?p& zHi{*n1p`;>-hi+r4oC#r*mF= z8?Mdc7ACDVe*VV;K?_uLQkoWqb@SpPDS{y;ca_&Vf&Nat_V>l*yBoC~1MI_Fer~?6 zuz87gTDOmAePUKSmLTbOp2> zpa{73bJ6=;O0!--!hsG500sF_d-<#X^{QYm#`37BG)PTF*TvQ0X_NAN43*aLBP;_E z8ozY|cF%&>OaP(gK5Pcq+_<$rIanif?-LaRrMiahQ|xQZP5Qg5h!|Akq!J@*#(}@$ z`_srpIAmQT^g&#mL!${;;qcz;y9>qHQP#}f0{_HbBlL3GIE!^T(J_1_@#tT*S4xAY z{W+*?LRWV)(cd6KLncCl3p3};d&9qBkpUwcrNJ*90Ct8TL#w_vfEC7s@S5cZ7qhI? zId~TMd_x!3wMz&DrlmQj9^-h2pwq387<_FmBYb>D$l%<3F^e76J@wa(^{NB2+)oP| z)y+@Jq~v~$Abd3P0uSFDdi3rg zWnL^+Sinq&Ayz@*_k7={c{0&}d+hf!qrbbczCev^uwT*WQ=%DgIP@!gTZxOd?W`er zFAWxMis^XKEe&qe=A~sjdOnCD^A9gf)W4ToOWJ<9G&`tN_nulZl<@S6@h-y!^-Ihjf-3_5ehBDp4q*%O#qb@ z;WKKf58}-bnfed^h_GlLpZ5Aym#m}E+@1O?E2QU=HB(0OiLLLuw;(}yAl_mA!U8~_ zZG3W`8^*D(Pw2(Qa>RX8CX7(9`E}k<=CNghALHNkoNqziaRs`YF@-Jd*Pe|HZY|$V zeQcY-IF`*hyiPci7a-n0ar*CCFBH1sGCggEczfBHNXpL$EK0}hCSXyRME#N7HTYUo zsH)$2v4%zTjtcutyyk;nl0=p@FSrE#yG!e`JPK!g_r`Gw<#zES*OV1DZ!1e^?eDA7 zX!PM8D-EB=Vz-ZBX5Y;9Rq&+nE{FY97rZ9wGUDPSe@^vuON5L)@V%AX5Z%#3#i2|?6Fk0 zvz!=yLDuHN-!rN{1xzERmpmibunptcvBO3w;m#ciMlXy;M_q?Y43WUrL-(U6rXsm+ed%+seM(n~2-uC{g>C_I}YJ zv(hd)bB`6__KIpYdAY@SV89a!*~jkPU5U@z<>V(`Y>RGyI3A(|h&{V6lMJ`qVvAaK zEf68DqPsCycs>~=cT3O@$t&oed;0e)Bs2!N6{IVD#VJUMcN>h*?qB3Vn7Kf$jHm6e zSiJyJ{u@BC?o)qHg#YOjetI_~#9-`oCrHTwxAy-hD# z1|(xnX^y*Em(eY;tib(OTnOnodWYn$8-u-$q(C@|UdGm75JYAJC50mTJX&Qi4)h|b z0>rRi^(IJR*8h7o6S|rNB?ZXGz5Zx&o1n4(kJgJX@0$Sd{niUW@MfQzuW$W^Y`}f7gK4Qj%#i~VdZTNh%T1Yk0nR3J!Ffw} zZVornlg(FUzCBqbYot}`Pi?Fhv$lHCYJM`xX5qxKJa3itUf*i(nT-`Y5zBY%KB{p& z4UMU94h61j>+FDtah=H1AC)lj8hyRWAc*21m#eCi!B?~pT!TAC}~lTwN=I3N6Vq{(7%_}vtO zsNOQ@h^ZUQIdiD6q%yDzH#1`Wxr=7hdnxM!GtpfN>@svKD(ukDu4KqgI6lR+Dq21j zDk{878Cw@cE>?>l6=>Oc*3=BXPllUoBy4mG< zAKH>RQu0c_i-iiX_$+XJL6EC&RU_gvrTuebJ_f=w*O$zmjV+Bpke|7PEdEr#?s0fT zd%}I%I!6!F#4^@>LwJGaJ9BPxc~E5!VgyNA-C*~|c%J_3rI@5>(B-zQQVE}a$p+za zfs`MG^b|oiEVk2d{%66(Lv6~~8AoM*VvC=0W|%k9?RX_5ZMf=1d2bylrzZ@_cVs6f z+Gj1cs`SmM=;RHGb!^WWUaiXEi~X!^CuyiyHyg7ig#B%*bNEt>N(1}7aTO`a=eGEA zdv<(tTm$i}VuZ$a2F{;7IAQvOKnBa0p^`U0DpT@b~zlkigE^b;~S5K|bGn&;q zwlQnErq8h?`3R*s|25Mwj-*))7KHOI({-&-LnG@ITb$cWAAx|fYRqD=+<=$#1`}T6 z%f4aCPi*V@!V{HOo;MUO)~9vk{6My?WJ`D*5;9BO$iJ~z@htkeDW}Wu8;;``d+e}E zJeMmIrr5095RH30T|1zz^~3$h9>{JU+u9a>U$L0U?(H)0Pu7z&-TYmGnV>wbd>DS(U&O2|2)g? zrKbCDIiFeC;IRn;j%eSc{hIlYsPN#Y4<|+KS*AQtyEI(#rZa2(PQm;T&0l`Mk`B)a zPO5UQW0NnqyPv%DQ6NPS&zW&V+pflc*tsHv(6f~KC>D}oM|QkjS5NBwXnhZxjM!12 zAv7=U!`suMf=l&nWNqJ&b2sfQFDg#U^B&kT$li@=`qP>6D(vo6&2F_&BI^aNb*(C- zx=X1I$&<6D9%+dqT=(s$Q*u`TMjOfJ3Y)KEiu0r_=DXYVky!P+?BP*! z=I2vil90}df>^90tcFNkg8#CwhCuAI;7zx8bNbm8S{P^e|RCrE5JFtLeRc9Zd(y}B*gS| zO71h2J@{HjzVDsyPZquVM2oyfb2)6}Y+6Lk`k(WanSRX|7LsrYd7ds?sYhM4bFjm$ z&E;eh6*cq>$`-0ggJRxd5Y{`KxUe8;GrRlIJuZ`(sV=j z%*Rg)&&r?5w6x6Q<=MI=3Y*#;h1VRuR^IM;ylH;HmUqX; z(s0mKGwse-Q;F*5u8;tU2&c(Q8Gfq+AQg*(Z6{M9AFJmPZWVTB$M`)7E;sBiq9=X~ z2F5cqeRTj;lFsOP>4urJ{20mlL94x5VL!M6u`0eR+&0a38_w?B35}7mk4xiOB7KGg zf#vy~m>oN-o(JIShbr@}MQ71tgl#Yqt$b~kgkK<} zA4U4!<*0%2DF4%Odp9)lf6qt+M17^^XLwaw_e0l-08?h-v&QBoZFYOHMHL7u9rcuj z6d$sOzSz`Oj)C$dQM_~H=nv8x?SutW;J5M;-4FX_%^pwPzbGeg zRd6hN>z2*7e{)>@{D_%!u05#YyqFHzA^bFP-+LBnKLrk^UQqJT#JK94n$nE5QuY+2 zK+hkcBS-5eF3&xce>c14S=ndEO=leX%YRrZyhgQ}>`-vJZC{Gu`v8aeMy;-qJAQ}A z56$Miw%nx8e-gwW(LDQf>Eq=;sRMKM``+&?R~W3yJlS^=nZC^DzS%2DOZ!vzVTI;? zDr19+@8A4~?|&gg_u!22mo@gAeT}~!wR3y0ugW(5@nd(r9iPyvLUcom`o+-DkhYxg z@YMf2Z$X2%{L;)6B(*Bjq4+%m^(?R<>CJlSpHv+EEAhZFZn_SrcCc&iA9)em zP2vAGo3BUyVbKhP_R#o zEuhb(vbR7;kOy3c0tZnX;3U^n{>AJqda74V&9M#_=!}3i12TjB(23upY@3oQXh>Xo zos3-i11!nPil}Khq4e_)YXs2{1ypsAn5@mY4-{j92{Fxa3Uz(m63j=%NCDR3GG*__ zzCT7y!{6?kBr#H^cXE%ho^_y{vPqgrw?aoy`h<|Kn!1n7S7Jm^<(BoFpDnm7l{32>rBlEQfu9b_SQTsp8 zFe5=Aj@V?XZaMZ$-{(#V_aHMYrt`Dz3_wr@GJyheiE<0@j z%=S5<&OKd;W55f&Q+#ctH(@Md5i^B5LCPqt^n*a~MFd(|Pa|mWypkLoUinBMbYYwX ze)%Z{SzejEJ}4L!GvTO9MWf6s%ks^-lz;y!2*I>L{1{-O8O8`4OChWY#Yo;Ual6g8;paVLY(>~tZ zWg=dF`L53E6X}}PM|t&sI9D~_UPLbAZ9m&4La5ii`4ksr4h4Jn;+t{*P=(!0iJ!{-(p)zIH$8 zr?@_YvGbm_sUVoY_{HTv_rou{C&oi($M+JNvimCw(c#sc}2g@ zhLzbLjzCW91Bn?*O~Dmm6+DqM!S9ZWS_1;{Uj3N3Dm`oFF(YCx?qTY4xXGMXtV_U* zH8yZ?K~#A5<_7ia`_NVpt_UQ=o;qe&RJiOZ8o26xbm??tj(M1su*%eF`{;}_DNt1K znrCS8Yv?-_wfUe3ibTIuQpDQ*O8NF+|Cx(EGR_kkWYqYrK%~^;VL5I zx*N1Z&UsGr3k&F1QI@?Ub5t-24k*>(TA$|Hr~9Spcym8j8NP9Snz`fJVCdXPX`&D- z)qvuuX=@-YthC$dNc@a+wnU)$HrZg?5>4p|p0SL}XqB4Ao-R;MZEB40^p4K7=&+Td zP)7~vGej=3ha5FI5}fmyCkjU&ggJIMSvXcimWP;Mw>jZsFX(%tetyurSDdyjtqEd* zTUb-i$&u#)Z?_F5FmIZ?qw+RO{ocJKrK=t&3-Jwn$un!J6U{oUJIDq8zPUVbtWYmU z(|z#XxJ}gMV<_}6pwROTg$92fVf$fmH>09I3{Am}eTJe7>>F3rr$5Ln&&}Tb6~1pZ z8xjpD-l{rPa}3@3CFE~*DWtqF{RsI1#uKN|ZW}~?wey~xdrFd#)D2-VUqz@ncV=Ht z^2B?xL@~zED}nQD6L-Y#yH_1NiIe-op#M|KSZC30S+XPQN|N6)*^xky*E29jz$bC_ zFh1vfpq#zlgg8EcuzCjDa)Ucr(jqb3$w;|`J4Iii&QGaMEl>ZPFKMp0Af`U->h1lI zG*bw2aY|ir+Bz|B!5jMrAhg1DlRGl@otdNFo5u)Cy7fuvhn{9_Cm7WG2 z)sP+lJ}Xh6ea$66rK@TrQ|9iGOLs$6m{xA84VV3T^csyCI~M$h%|P|(+LLhq$RGcL zQ6yi8^ot_bOW=J`_g|xio97%1VCMrQ>4YkS1qK zZ5j1`C%;0UuC8X8EgR&aegxm&wNFmEB2{`!USYE}I`xuGM!1ufa*Q3g52c=c-fGgN z1uN-Jvk~Bjau@@Ozq_-&2UcpuOFUOY#4j+vk()TE znz;JGKI7tx)e8n&BQ!gGU;25q(rlMkS9}SC9qGc>V9)+#O){%qW9jd5AylPrNwTWj zLy)`EHivENky5B@O$q+tPO8*_=Dc>6d7D^-)PJC44U7mU$K7`%f?4?(1!-7 zg_^R$O=a%0C4+aG94aDe?TXeup_<&L59FtSbP&34DvJio+B&x5-|5yvbPtyW`3+q3 za*9o`u~10K{gSTAKrQ}Ol48i5%-Xt`9q#1VD8|?sywF5gEx|=vdX~)G-aoY1$ujD1 z=bFX9)Q3G z?F|X?;xLmc09g+6u^xS%EGO`Zq39Uiw>iAs2+SpwNMwD>cpYN z$`?$Sxv;P06M3ZO9yEM=NS@rl4WI3>*hy#SlX&LaWMbJqNPTqZqrV}Y7r`;Jht9CV zUFvlSGeT?gbYI&B<+903{Avh2I%{O<4k}g{L%SiCq(ORX_khZQ zSz7IY=Kel8zB4zI-h8dcS+=iiunpyMWl>3?5#z`m`=dTxk`0X66~#M6o)xageWocY z-?vuHf`uafscFwl_~R&6GZ|$)b*VjV9;okKR^z6_tGLMUO~-4H#(!Cih^&)ryr&yG z1rwB1w}*&!s8!l$_r1QGmfkNCl9xWESvAza`2>>vt7Z2AVC|N$@&<>r*+FM4oX#Kan_4m;yTP)YP zd22976z=&T0T*o{bUhQ`Q{mI?q1R2g2K_6;uJc%MV786n9tHHWQyj++5R~}ws zYv3Lwuwr@p8?-!uHwdXF35V3D=d&pKId=3cKfqK5SQpM_tMEXGHEoB|JYzl%kQq`I z+HRkCkS4=dCHg5qVJvy>)pW5&V>yl>bS~vb-F$EPSuHlzZCbn!dF(`XC;6bOGYjTW zmb}y{vdVL#6jVFD01)t%@G#;DV{;PHjB+>Kp8tJc$%u3YIZZn5<*=G)A z6o=Z|dpztI^>jD}jDn{<7`; zCLqdmD1&`vI~(k~JX15A!hiY>83|Gq_%sQbY5iM6+A90_k$qt~TM57&1es0<8UKC1 zrdD2?;ZenLNDh*LJ9AyvkHD}9sYZTL=!^-So$1=#Z$*LF5#qk4=Idm-Amu}2+bjFG zXdQbbiAUzGQ+!bMz2;e3`==5@idSS4j8%Wuo>Qbyp@^Dc7Pl};2M1}x8?RpbQZK!# zNIPt3-)cT2vZnjVu=ATrQA^&#*V$)E)sGx4uD|8SgQvU{f50pyyfSDu?agpyp`JK$&0pHf2}3cBx*l`Li@o1G3Y z)c>W+ZT^R2&{K245*1U~@pZOa2DCB+E=76n;Ct-|=OsD5$a?l~^m-XF4ZnW9a zj5Mv`G5M>F=eAoil})5=NJvmWYCq7;7k7lzvq%u8U##9oEzQT$uV9%&Ac1FrN`V6w z5|IDB@hzC@L=YA<5O^A@eDaS};ZawBS79gy(kV!HkAdY3AT>gr@P14R|g6Oo4F_paRfP;vqw%M2Fz^)Yt>!A&G zDGUzN*2Hy~t_1`k8=)}{#qU4g(X)R1V$AHh?lw8ab;3B*^W1f*B-Z*OPxs(_dUt&& z*t(-#1DMpx7`Wfn)Gj-FgIl^k@4#arFt|+IUSxZ`3sO`D<$&1D#7&Cs!yx@Q2gMO-Q> zTc@a#HuoXifzjbUXlAXxM?dsDm$G;v@z-f_)x(o@=*X@oT6+xZzRVoQZg-T;^ZI=k zPjOct-0f@>FzG!%4pA9b{W3`YF4^FZQm1zH?rxw>6kEzoXa1?{^bS6?vHB*7v|Z&M z(^M!2M2Ufhb#wFg_qQn88?5`_AxADQ4sUy)^F+Y=_ff&+guO@Dnae8p%0aCh=y1+! zs9Js1qaDidEUEXKIwvZAAI`NrdOotF*3Mhyl5PBtbL&H+cngTDq<&(rKmp45>~BU& z@$0}9(K}F=Y(Z#?NDcNcV~T}BG)P`hSFR>caXkhZd3T{1#p0qj?Q%xcVKNttD%kQK zFzu+42WL(-GL1>Ier|95rSeG{w#Xx zyf|?aXGi=mj4cNMW7}ICk`OFe_-o z=f9v;dl$G?Ros$C3!^Iux z|2qX8>j4ASs4QT%=P5Eq$JC+ z-aeFbADq1^sVyh|?^%(d>oMPJv2BfQOs5l$rwAe^8et2VTEfr%8q=+cnz-a3UJMyd zwT?o0Fad@!Eyg`Yq*&{Rj5=7UTHN9k|DK7iS`Pm-SXaw}fc>_XT!Rm5$5&(t|@F0~LkMh?x8qJO0s2k{*043obItCxu&g4SeW6 zD%E?Ohbm3)LEEJ4Aw?+~RQo?4c4YD_p}!S`KXTVd%}#g1Y6zy!;$s%pWx^{V_R4(- zygmyd2$2T$tkv4+$mw8vQ(Z3bNKgc9la7dT3!+fsRa9-?MXlT-J`C5e8*xcHUh$kT zDD3Wt!bhUk>gf?xNEiHSx=@F}oFbir{H_aLT7UCQ(?v|h#;*D&$xec8VDEtwE7ysxMK@3@4wwi~d zMDVIhfBX^r))!#eTX9OEQVkN!@xnMkT)|;c7VXGUuM+kXNrra(dT{CvW$u+218>Pn zm~CPXiq%#%oX43=-BBoSY(9(u_(n&#Sf1HW{*>6jg8bwKZQ$q#@`o+vyUKCW6|6$% zrJRTwV?T?O@YwECR!}bhV^kQf_s0!=Un?`9bw_t8{C2268Iv>F@Jr%v@kh( zaqilM$Q~_w;ROTUoU#d9&b)8!obLSoM5(Xadvt;v{S{QLmD1=F{F}5DFgrhWq zluL#CmTf_t{vZv+ACi9#Oiz|G9qEq3W=u?AbDZmKFx)YN`9o{qo+E-J-SepX) zO@Dd1i*88&2@`IcUKrx=>bfCh0Xyjd$RoKK>#rw#fKeXDQwbEUXf#3hB@MjT&{rnn z)Z|!5^F}wt5D5ZW{EDHLX(>ICH9a0@saar6?HcdL-5j`U6P2PH%t|s3z(6=S~m+Op;_0$GBMTv zKZhW-(HIj-qvvmhnjq`0dZC}G@n+6^`{aZRS-u(kz7}Bo!q;T@gU}gf zL6Ld|Dnzc%*21uW9^63N749*~NTnN*-8xq%d9ecu(2bSQe6k*ZiscOoU5W7WkMV&V z$hL-Z!!O|d(PI)0At6NnOUUB_*MhfGCIQaJ_L~Z#eNd!zDEKgOlKURQR}n-&T|ILJ zK-Bcxv{)VrS7?UZ^CYRCBUE48El&ueSs&mJ7|H{!7zuD{2y>ho{LTu~M~2E0P~*T3 zC)&PJ*E?2ccQe=DahGkW#fp4NK@^CCRdGlW%w`jIGdTYlH`}nF0XXtG!sXpLqr@p& zy2^vVV)_|gtOer;oK}WIKmt^6-<~{q$w2eV(*viiC!=<`QlUb3$sV$`q^jiThco}u z4rR!s@DK0y45IUNVGz(AxLyZd2^XNWl>ZJ&1B&HgdiTb`*4&^m=iWs zj(PHkln5{`E9^XdBrE=C?YrHi?KT8wS1mll!2jm`DBT9QSv|WaFBxW{6)iygPAARy zD;yu8Tb}~Ykb8#389`W^bQFTNe%k;v#d8zbKItfIzKC%fcvF+R%f#K$UX=B$9ZiIf zC=U_i&qwP*hH<=U0Z>n+sDuI7p+zb5btq{|%S&KVMQFK$m}l_UK0w5JJcKQKg>q|F z%OG%v(CWX>M!gC{U-%AZNTq}q|ts zCtdg23p>j^N#B$($DY~!}YH% zQ>!G%x&S2+#Nj2xw6-v2<^m;K#MKF~P7WHEP*oJm+kCaIsSuv}uzti`;YX~)!P-CQ ztQo5jl+i?&((-uo>6U)at6_vf2|C=0mv$C;P1nW~nNa&e#C1R`1yUesp-8y#9W<%n z_DX_9UfgHzZUNV_X5eB_SJ|DNj)A${9D_Q&6Q%RtykuE0K-#*EdKu(Q=qZUU zGUIV+7(bU>+okt^@(IymPKQ~HCw0GTKQXf^q~amgD>LOaO_maTo)h#--d1i1rMMWB z5gz_&X}wAf7As@zI9Jg1RUR?yu6y*F>}M=HTtRh!M@-d}>UNj1n2Yn5qPQM@WBI_|J;V@iE++=HQo`2J zY?6W73yoJ@{KLnZdiU52$mb*JV}u2S@^y>xy(8aL=&@7&Tv zh-R-y8UgDr$u2_5tMxzdfgB)7mYC>^TR2_HZ)%j(U#k?bXstX|?urp~`5z*Cu2a$l z@*fhEsv3<;>)Sl!)BdUqv_xC!_1Y@>X22|a$^vNB=Ut;cqvNN+8un*aSAkyL%R8OUYIw(l77QGECQRY^$ zW}=X@FM0lsFROVg3h127h+8NwwC0kncc(Jt&`=KRRg_AOV4-fYW|G_42Xf*2m}Nku zPlcsJ#*u{wNVj+ls1hPsF=5}Rhn7oZSI!gEqSe(a!)OY4V(r%E-*$lQJ>^O#T*1?D(a$MhD_!K z5ZsMQ+kdBQviEF6-3IJ$w%w%qLrG?yxdsF%w0P*0(U978W}pXu7^dv&4hd!BwK>ZC z`Tmcc3rkJ*$|Y+kPZV0?+1Fn>`@V6OAzfsUnD9{Wmgt;ygQ4}jjytcAY1ASP#`o~! zi!Zt`IA&@kwSV+~EkwgRQO;~4hmw6(|I8QTT|L!OWs>R4HlvLy6+#LGCd%I#)G`9M z5b1>*Egqh42R9^pcj1&#~nT-9D{xBDwWy7ob zfQfij_jr}wT4_KUpbS2(@mSadX6{D827T$X%Kn;fSD0r(hbLvLVH)fhA5ATGy=R*l z-(T>ps47AQE1o)w5#sKVX1^nGS?bdA8|TxO7@?k@j6UxGj?Rqy5KU4UD0o9UslQ&%AYaUQbq_vNq@+S~7!oL-` zVi3-@M(=O+raLJ>-@Aw$5BJ82lPaGj1e{x zrS*R_UZF5mfqunpM$fs#Jt38KbiW?%(fyJe(<-g1=2Xx_bfsW{W%#IzB0a5z=+K`mi*qQ;U&KW3dx|(3e5%_G!C~xu>1F~^ntCj=%TzI6^1&Lfq=x5-zH4bo?^D< z+slcr6pOJaF1@;&6V`c=F2Od7ms(N#vxQWpk{fQc-1QWX#6&D%RktjJ%>b-(28+;S zMy!p(p5^Z&O19w?xyg553%PgeuH@bwyrBgY5mu8gxgWWF$$;n)yt_51?6Pt4!31^l zQcSKwK2h;3}ZX_jcrdME&t5|$os@4 zS977~^@;dx64h>}L?-w?712B10iPvQp$;Tf#5aEQX|O0h6$E=v_|Wpb*pdz_R%YSd|}+N02YcbY+_7tz;vqTJkxg_`c4B(*1 z*@^NXSF!I{tgi|;zz`)q>ta5`(f*}RK00!bh01~mvxR}A_6X}X3VNOiZw}j_1KC`& zX?GS2Z^x;ok>mG^9^F1n0pdG5f+|{Fu zqVsooX)c?aQ+R-qNdJI8^T7IJSGnPwO2Ifpr;~r^V3xkKhhZ5W1BXCDxF@`iV2rGL z@}A1Dx$p9X?<C;}p zKL+VRV%@vv&)KZz8E1)G9hO<;9Wj3KwLYA`2I;Q_Pid`MfJD7j^=PW~Qxo27Gy2kVo&S1DITSb(4_Gr+d~4*Ig=JLV(&Xxx z3+98W4VAUXV4^1q-hu6Tzn)`4b)-pmFhMeKxz-jj%ZJY+ff;`KS1+B-e^dI`NeWV@ z#A;*%1DnryCAe%`rG%lzPvr!(yqNQ7unOlnqZb{x$aPYW5(h82l{(!DPeb+DBH~kL z4%l-T&l!pnFP#lLvrTT)GfzId;1++E-;6_S>^U2<+@sth=={akiaYmPV~^_r@E(GU zDVK&JTpg;j0Oge%tWiOkV4_=wtSh1zWk5El1?9;9Lemyr2H*UwVW!%8;anL^X#E#F z;-zTB38qTJ}vnWmfl{5WJW;LwWUWge4vQ^?%&F__pHrtt^~H+0dap z5y{xgUNzRu#Udxn2#EvXTPJ#2DPnt|ZL646kcgEsolj95NoWGRnmR|kv?BK%u z=EX#p55udepwXH(xoUv~ zuUOA2Wqo9*lYKp35((1{5ru~}X2T-ur3`BSvQyi?K5 zb3w&ibN_W?%t_xaPsql|X0KxJ=*%Kqd}LAQSt}iqb2;$XV$s9<{>;`2Z|{M8M>&)Y3hK#1} zz+#!xCuU*`DyW*A?z17uE@sxY$t8uJTE9`UAc~-|FobCJg-qyF#a1SN&%;4*E;l&z zxg`wIjfcWJin3H1;ZM0>cC7OcCb;QNpJ4piD>j)Dx8S4T4bFw#?#Uj+^_%X;)Iv`^ zp+LgzG6LU^h>{!5$|rZg;}-o%<{{E^ricU7dp7NYYjDn2RYUQ6=oxQIqZL*%+u7p_i0EhC&M^xNwK7N2`k?4^$ ztrJDb553O+l=KU|CBnb}p~lcfbCui(GEocm4@dXEHdbePBk+CSDqqUIdHObvEZ2h@ z5-FYYg*d@`QSBC!BG_T9%S#H-nfYBHw-`$b}bsEN>vy=M|78dI()?H!oz#B$`VI6Wd+qbAT#oFQIu+VsH66yOQ zf#z*?#gER@3WY9b7sdAQ4Pzv)c67Uq8&`vc0CV98Z@z`Yi*(UpX37e6*8 z^a;6*{=E=2D7j!TE2R}k=N-{G-OA;!{_KmlTHtNSHxGL90$AejAw{C#(jzqrw48Io zRlak8i%Xdm3R2|+1fCx1E$druypvVcolWKWu4iMvc|}%FBYS5#1Rm>M53}dc+>3?} zeB+E=xBW0Lc6;i#$jBZ%C-A%X&SNFy4uaB56OYyCoj17X zmjmI9|2ICHl$4?B#)ze=ui;eti`!@aG#Rg!+O!n7QrH7V7;6R_(pIX9+fbuVs^&Se z5SjDy`|alhzbW?aqVR7X%2T~vH`?BUeJ;D4VRiK$bM`gNKKMv1qGYG%c%BfOwK zbz->?_>-IdF>$`K3Zjq@nsz=Ll|XP|F6KU(Ya2QWZnqwGM5lyTri5Em%4}pkS%s8p z zo6;sH;tm8_i6ul~O%2Nkm529d{NgR;af%v~>+~4&bRPhXX3>Ulh2@M$|2p$pnS9Vz zlt$@IZa(a=ALp=B$4)$IAY2_*ezm@u@M^!F^8NH+x1G1)1CaHZKOEI_30r-WtVg2i z?Oxxd>+-%Qj^`vMU}_g-N6Qt<1Nh*JzGq%IkPbAiH)DJ0o+{Gi=xHmMRX;xK>vC=} z=Shv{2yf}ipkG=zBX>%0?EzM;wI7pS#ksQ*6Wx;jiZ0fVN?b8s>s`SjTbj#Z7Abi| zS+f93ZPWt=f5??A+7s5b^zr~jUg_N|pQigEP|Y1o);oD0cbVCJfPwKLw$y=CbEWM! zWc>3n0u1%146c^ZvhkZT|J1>2-d|Hr8e|h$%|9Hup;b^u^Un>j?aSPkizDrsS5+7m z+I5v=5_5gm7%)4$w9z?P!kwD0UqFZ@nhT*6e`EB841)#=)UbPI*Bg6>rAoKn8kvRG zQ=+i=F5J2j%2LUX4$Uz$IA0-CzjoQ9ceUr(bn)cQGhX9ERJzp9CuMSYscqA}mj{)K z=O%>TJU&m$mKxDSYKZuH1OJt7NEu6e5PZCfOF*rzD3ly9{h+vNrHh~5NY3wjx}^2o z&USI9(R1pplXS91{fI{a=9VD{Q_FlOeT+EQi1MzZ_i2zyiHC6)+q98mLTxHa8ncQS zbHBdp&nMk4!cUUSm8VjvDZ9%am0%BFealOE&^%6uk2*MOWc+Sp(pQxd_SKc#Yys8uOMH6=v~rE)eIFe_&Ln*z>t8!*u5%I8(eGb=7%2+W}fl4On4c%|6K zwpKrRzEr-s=`dbA$Dr+>i|(HukRSGgSD_!}Eu9RM9-iWi=O57ArHqZ0VLU zx<)2#tZcaHI7f%v&lG=3BX4pjMbkB3c?(d}QBL+M(;~;BcY^E@-luEh4Swh5<(B3&)sV){*C zdY3z+dfOXE{q`bq9Dtc;sJlFG?8skl-!j;Y(`an}U+rD_Kh)jZ9~6pcR7`fGvWto= zV<|>a_Og%lp6p{MyFzpmijYKnaMO2@dtsmGXZ3@aSv>Ck9vxd`o177vSSMIG2_Ul^34Rng^pRO zZl;sZil%qGNSjLHk3x29O|FI-vHGZMV(XLU<%fZ73t{oG0>(OEhh`JLD6*d?26CYJ zrsYS2O&)EmAf8AQmCh~rRGBLhi4q;WYnSk$?_AacX=mJ5?{3n0-gnEGV4G%>+?GUdw5r5be8fo@A7~j>pG;~_vV6H ziP!^bj8j2Dw`+H9u<1*!N-wn%%kHoa>WJTtd^sha^a^SI#+vDju!gTs6$rRNd0p(Lc36P|PGg#*`<^OCvO2C&VA36x*}`z0 z8>Q4=?oeo_YF&9zbe+j$b_@jONv}vTH;)(aCe;tZmo$7umOQA43pH<#G>i0oVTori zpHae>jB~0vf;2wMC}Vh-?mnt}M?P;uE~u=^SFj*(2Bq0$IeJI3e$ly{r#q~Z0`;eR z(Yh+XVywAmeI5SgXp`1zX6Y-n>-JD6`nHwdsdZ^+IO6iU)B3oKy&&mkai`*BimPD^ zzS|c%uHbWOylzrA&h;TR$eZZPp%TJ)6r7)Z5+WWK->@hu`c=HRL)eFrHw<;e;8Q|y z0&+WGM+%-ww$yCe7qaQ^{1CT}%eFXL2gkPuq#P0gl09vG0l!o-ip(S;eR;&{o+k~v zJg5lZ6YPMM{XDJs_=j|_EWNAGh)K!&&#}uHADzic5DS?n%b~$eD^;N5p~lFa#ii$V zWWo9?c-^|3eV)S3KGfh<`E{7Jj2l`-(4R+T;`7D&)Vq{fAvMRvmDXQN+k}#9yq1BO zdwpA@TFcYJ;eu&>OHWw?SLP-0@95Pk!#sUYp;2BOi3@+%dVsQq5ppuzYFqAcF?BFW zWUjR$1ENOduxN0Q0B+nH)>-XjYky+(S>`%CZ$nXs)cn-UFJlh}f;#eQ%ft;5`qw!Y z9NLW}^{k$Uc;<|$SRT?9B?-}UU(#vM;a9pJgdT?Tm#|u9;nO+sYP{`aSoKN@W#JZU zwesD+)jY;7=6?HZb4TKRu%o4Wp?(8ivfA7x((H*q++0iIHOA%+nBQ3x)4WjOs~#6F zPf6F9sz*xU!7lsXof0mk#c7p9NKJnY+>w8b>2kpo(tsUNkS(4YU8?BK_=HttkJs$p z!N<#Lm=%6&jxnQlh4-Vq&nAwaH${m|mQ9?$G$p%tCdcxe@ZI9~>6+E1Kkbuj?C31VjR5Aj@v~x=Nw@BPb?z%t2+pZUfp8hXLrmHadu3N zLq!j-sFm=|RwcGJBH}a-?0R@d0|ll-)n-3uB?v&nRSW$ABlQL+GmI{$PHbl)HZ-oI z4~Bl(6{Gf{Hxpt-uazPZ30IkF(f7DquMQwo%trr$ES?8P8pKscVbKBjb2TS~(UDup zxHuTsc@CTGTprv!3;;)C0Qa*rLP7z2^;~cuNj`eAOv!3%1ADD$`axtYhBw5O_eJ6U zu+>H%DRUjbh%Vw+kB>Dg)Ja(T-14d^oFxU{`at@W zZ@}hLo49r)DLXmWH)elYgun46G;zA?7SoNo=PEUu47nErcU`j%D27iqX(QF2$)un! ziSvT1#3|8n3o-=AjY|;{Ez*kQOZd^q zqV_1Y_O!h9(Uw$wLUZXuhZ0{2?#^}j-xxc6tC)nxJv*y2RIZ|DS(U{PBtx*3-W-RQ;-!dQxIIN5PUw0{fAwfIWP=vg@O~`A&8@%W znlds$zZg|gapXtu?m&~Un)hZ*tb^My^GdAW`6QLc{*xol@iZ&&=qR{xT$qRO-W+jC zk19Q?@CD-1Tu9e^RIBkjcR#HX)DO~lSH6@6udg+Hg*? zIcV5)kdAwZ)<*+u?fYB)q2eT~fbgH!ZC~>@2(_!3Tk?EOiFiZfHixe(N+}$RyAI)v z^#q4RTHi8!{U1+Xap9uxrT-x_A$Yo?)>(6A6zRz^IZZAY<`OAnc=R(^(<)*AcsnHp zSFa6rnO+Z#`*IVb?19?uEHvAdL{F4qj-hL%q21q1h5o?m7DBu&EKv>q-*wszt0dQF zDRVGJ>dTs6@lUL)O7C$r*zrD~Svvuo($F>I^!E&&kOn)V?DK*o3)NxI=SCSr()6#- zG}s}8PFDcd6b=U#jC`|ecJM0uEKfB~#qY@Vj3Om%&b%;f&e5Kpwr|kWZ)5n$H4{2=e4W@bD zw?BpFtKS0iOro(gVFL8+ys@--(WN}b7o4Y4h%Iy4N|bpAVM+zosr)eV5p=NmfI6an zt7FWYV>Z>{mkK}zT0sZ7}28kb7XJ(5EOqzs=A8X{7W6^wV zqd)qrs!ns$5JOMfyx?K<96uv5YpB7FzLl7R3p>;Qubs;LrTQqNIL|9K!c(DviQjJy zO$-!Lan5N5!yng+!+q|L1Iq21=pwe`^SWvwdzmoJYQlXgtPvV!EWC$C)tt5pjJ=={ z3w%~AF963v=cL=cqr`iUC)dWiI}IIAX^U#i*z^Lgf2UJb362HRhvK&QFH}U_w2gPGXz%@tIlEtuPLTq@-_E9~j zqR3mFD6`fn15`XGHH|)ULk@3{zB=>td5^4*Ld8E;1X&TXdk2BXT-5V{V3ToAm`H~Z zCPVqBmdl>OfBX%pNI`c8a?k*fhHllE0^tI25{uq-g~A^G++aOUrr)(}Yr({%5OzKZ zeT+L>K7-^xssV+ZvNeBxP88j+;k;HLAKs8Y=JJyYHGPbc&I(|adp~BbOX9oBmGsMn zZ;iU3U313sB>=BOqX1fN2$3?{nUrWoL_0w%3cm>HPXyyR$LH}pyyWKkk->lyGnnTs*A9w*Tbw!G^W(aGKfxsR# z3o++0`sJmissUZhstR27d2b%}FSIMu&FH;WvNT+t4SgJJ={rWF`+6GctPv_H*wTS{ z_x2yhIfC9v&Vgl3P_b&_jLU@}JcWAz>+R&B zS)IoMM#BliO3>O4`QpOWkX;nL7tZX^n-uz3`Zq!hWbU-ZFc?XW7^ZlB^jzf6RYv_R zKZ7mBmCnR?Dq(v}@?CoT?!Qjdfqi$S0Th2$#xHNZ=}MOr)qZft$(y*as*qczc&LhB z#$Dm=;=m3fAi0dFLy42)iz(v2k+T%-31+XQg)|kvf-!wEJS$*7rs1z@|G8 z-mRiTl*oDz(sTy>VOdMYWRMrg8Z=?m-%PPN`|%!QH;D(CU=T zNqYO*wS4vT-iuPAHu-4)0dM@@C@pgujS1ve($DM&5(J1RZ0nOhj@0U;95Q)hf$s$+ ze^(>gLEg9}0kmJN^JK;&TXG-V=NKy28V+ag3%ys2k8}?~&A8GIs31w2sD==~oS}WO zcPThlS`d3Y5|ur7PFTjsm9DhzMnFfGR+Dp~(zWAJeQ(P~27e6#fZofvnmGdWwnVHM z51Sz#Rb!8-nUNAFI3S-bKrv2o39J^2$W=x=RM*@gB!40PP?NQD7g8|bOxr!#*EzqtftJO@TS z+jB!D9=!JagfgxBuK*OWy{G$TeET3(MRR*>JZq|ohP69&C386ml^#^ZVZ5B|47uiB zmO)GTuMpVcu+)yV&k^`SD;C~Sw5H^n@t#-!#npZ}5iu>gQWJ-Jf{bC8-#!UlpQE?K zcP4a+3v!}vx^c^sC;MAh-|mpJOoXOmV{vlJ+RS6G`bqa$-@)Rv-tE2Y5e8M761z{8 z@o(BcdBWWv6Gp7|l2uDCpbO0gyMClE^~PWpjmU_2AbPQPyHEGK>C3%j&;FlD z{JB(RLLvt40cD0oiHP`#tbK<3r7_j#BD`V(iwUJJ?@Dw;*7U$sjowagrKFFKZ zF1m_R56OwbHlo!xXK3F%fio}6*4a3*ub_!mBem294#=27Yo3*vyTG%7V$ zn7E#eE8AiyvH0egQRK(*GV1m01@9D!6y6bI{wdv$7Bv}ggm!_oR;|Nxc&<6^^rpJm zU<4y^S=e6eh^GulH~{}$GPl_<{U{7TPdtxS0}xIH5)X2}btxydHfL+cyV6xu#wm(? z-R@Zg;+qzrvwCGz_Slm*RG?BA;p#iv_vA6Q)KQ*OM0PKbj6(2Vxa?n+4E56witX3V z>+`D&*Q}dNWBl==v6s?UA%` zrC<0lV!0$FHRZKT`bJr*JD`LgI6~cRlsNAABAVOkk{Bz8^x-B}Bj$0dn2)(Hu2dn&(!g$?7bjqsf&HYOc7fmC{ZN$K>gre8ZI`zlO@3$f^+HRIg5ovcr#tAF#woHHK&GO1 zwxsx#Bcv8mj`SAeN*!v2$$<#QC#Gu4uC{J+(vnkNw@(6kU7-|L&~SjQsJhQTKK2W> z;oxv6gs+;0ckDj=65wLKkOlfej3qnfv*)Dezxus$Yuxj6PZ7}BuesI4wfZe@u24VU zTg#*j$ric~u(&$(3k+pdYj1?V@@XVBz?3c!vpw~puYPwXZN%d^`RE3K3i%J&F&P}6$P zg_c!tV(-tBQg{d^L*aY<76DBdruU{o(|(=UgnDAl)>zS!lu9V!f_~V1pj-IT?^{&u z1&6`xpWFlKGz|pNWQZODEvdP~KunJA`o4?vK_1jhiYE8pr;l?+KOO z{CBHu3{C1NICDU&yrsVFru$$BuKJ)Dg#s|4G%tpxYHT9te0Jn;k8UK~*v0{Bu#K)O zx3=5S=FOVA_`d^SJKW4Z+!LW;*(-2+Zj`|+s`lB0uK@I_2GG?QWoU9$1ld%jkR*le zpZa8(7Eiw(B5YCl{Q;I5^4f+v5UWsP(YBD@~ literal 0 HcmV?d00001 diff --git a/workshop_public/RTA-mini-workshop/dashboard/static/index.html b/workshop_public/RTA-mini-workshop/dashboard/static/index.html new file mode 100644 index 0000000..c147ff5 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/static/index.html @@ -0,0 +1,83 @@ + + + + + +ClickHouse Forex Live Dashboard + + + + + + +
+
+ +
+

Forex Live Dashboard

+ Powered by ClickHouse Cloud +
+
+
+
— ms
+
+
query time
+
— rows scanned
+
+
+
+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ + +
+
+
+ +
+ + + +
+
+
+ +
+
Ticks in view
—
+
Last bid
—
+
Range low / high
—
+
Median spread
—
+
99th-pct spread
—
+
+ +
+

Price & volume

+

Drag on the chart to zoom, scroll to pan. Change a filter above and watch the query time.

+
+
+
+ +
ClickHouse · Real-Time Market Analytics mini workshop · dashboard
+ + + + + diff --git a/workshop_public/RTA-mini-workshop/dashboard/static/styles.css b/workshop_public/RTA-mini-workshop/dashboard/static/styles.css new file mode 100644 index 0000000..88c8f9f --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/static/styles.css @@ -0,0 +1,102 @@ +:root{ + --paper:#FBFBF9; + --ink:#1A1A17; + --muted:#6B6B63; + --hair:#E6E5DD; + --accent:#FFE000; + --panel:#FFFFFF; + --up:#2F7D4F; + --down:#C0392B; + --display:"Space Grotesk",-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; + --body:"Inter",-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; + --mono:"JetBrains Mono",ui-monospace,SFMono-Regular,Menlo,monospace; +} +*{box-sizing:border-box} +body{ + margin:0;background:var(--paper);color:var(--ink); + font-family:var(--body);font-size:15px;line-height:1.5; + -webkit-font-smoothing:antialiased; +} + +/* ---- Top bar ---- */ +header{ + display:flex;align-items:center;justify-content:space-between;gap:20px; + padding:16px 28px;border-bottom:2px solid var(--ink);background:var(--panel); + position:sticky;top:0;z-index:10;flex-wrap:wrap; +} +.brand{display:flex;align-items:center;gap:14px} +.brand .ch-logo{display:block;height:26px;width:auto} +.brand h1{font-family:var(--display);font-weight:700;font-size:19px;letter-spacing:-.01em;margin:0} +.brand .tag{font-family:var(--mono);font-size:10.5px;letter-spacing:.14em;text-transform:uppercase;color:var(--muted)} + +/* ---- Latency badge (the showpiece) ---- */ +.latency{ + display:flex;align-items:center;gap:14px; + border:1.5px solid var(--ink);border-radius:10px;background:var(--ink);color:#fff; + padding:8px 16px;font-family:var(--mono); +} +.latency .big{font-size:24px;font-weight:500;line-height:1;letter-spacing:-.01em} +.latency .big b{color:var(--accent);font-weight:600} +.latency .sub{font-size:11px;color:#B9B8AC;line-height:1.3} +.latency .sub .rows{color:#fff} +.latency.flash{animation:flash .5s ease} +@keyframes flash{0%{box-shadow:0 0 0 0 rgba(255,224,0,.9)}100%{box-shadow:0 0 0 14px rgba(255,224,0,0)}} + +/* ---- Layout ---- */ +main{max-width:1180px;margin:0 auto;padding:22px 28px 60px} + +/* ---- Controls ---- */ +.controls{ + display:flex;gap:18px;align-items:flex-end;flex-wrap:wrap; + background:var(--panel);border:1px solid var(--hair);border-radius:12px; + padding:16px 18px;margin-bottom:20px; +} +.field{display:flex;flex-direction:column;gap:5px} +.field label{font-family:var(--mono);font-size:10.5px;letter-spacing:.1em;text-transform:uppercase;color:var(--muted)} +select,input[type=date]{ + font-family:var(--body);font-size:14px;color:var(--ink);background:var(--paper); + border:1.5px solid var(--hair);border-radius:8px;padding:8px 10px;min-width:150px; +} +select:focus,input[type=date]:focus{outline:none;border-color:var(--ink)} +.toggle{display:inline-flex;border:1.5px solid var(--ink);border-radius:8px;overflow:hidden} +.toggle button{ + font-family:var(--mono);font-size:12px;letter-spacing:.06em;text-transform:uppercase; + border:none;background:var(--paper);color:var(--ink);padding:8px 14px;cursor:pointer; +} +.toggle button.on{background:var(--accent);font-weight:600} +.presets{display:flex;gap:8px;flex-wrap:wrap} +.presets button{ + font-family:var(--body);font-size:13px;cursor:pointer; + border:1.5px solid var(--hair);border-radius:20px;background:var(--paper);color:var(--ink); + padding:7px 14px;transition:border-color .12s,background .12s; +} +.presets button:hover{border-color:var(--ink);background:#F5F4EC} + +/* ---- KPI tiles ---- */ +.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:14px;margin-bottom:20px} +.kpi{background:var(--panel);border:1px solid var(--hair);border-radius:12px;padding:14px 16px} +.kpi .k-label{font-family:var(--mono);font-size:10.5px;letter-spacing:.1em;text-transform:uppercase;color:var(--muted);margin-bottom:6px} +.kpi .k-value{font-family:var(--display);font-weight:600;font-size:24px;letter-spacing:-.01em} +.kpi .k-value small{font-size:13px;color:var(--muted);font-weight:500} + +/* ---- Chart ---- */ +.card{background:var(--panel);border:1px solid var(--hair);border-radius:12px;padding:16px 16px 8px} +.card h2{font-family:var(--display);font-weight:600;font-size:16px;margin:0 0 4px} +.card .hint{font-size:12.5px;color:var(--muted);margin:0 0 8px} +#chart{width:100%;height:520px} + +/* ---- Error / empty ---- */ +.banner{ + display:none;background:#FFF1F0;border:1.5px solid var(--down);color:#7a2018; + border-radius:10px;padding:12px 16px;margin-bottom:18px;font-size:14px; +} +.banner.show{display:block} +.banner code{font-family:var(--mono);font-size:.9em} + +footer{max-width:1180px;margin:0 auto;padding:0 28px 40px;color:var(--muted);font-family:var(--mono);font-size:12px} + +@media (max-width:640px){ + header{padding:14px 18px} + main{padding:18px} + #chart{height:440px} +} diff --git a/workshop_public/RTA-mini-workshop/dashboard/test_app.py b/workshop_public/RTA-mini-workshop/dashboard/test_app.py new file mode 100644 index 0000000..63ac421 --- /dev/null +++ b/workshop_public/RTA-mini-workshop/dashboard/test_app.py @@ -0,0 +1,78 @@ +from datetime import datetime +from types import SimpleNamespace + +from fastapi.testclient import TestClient + +import app as dashboard_app + + +client = TestClient(dashboard_app.app) + + +def test_dashboard_rejects_reversed_date_range(monkeypatch): + def fail_if_called(*_args, **_kwargs): + raise AssertionError("ClickHouse must not be queried for an invalid range") + + monkeypatch.setattr(dashboard_app, "_run", fail_if_called) + + response = client.get( + "/api/dashboard", + params={ + "pair": "EUR/USD", + "start": "2020-01-10", + "end": "2020-01-01", + "bucket": "day", + }, + ) + + assert response.status_code == 400 + assert response.json() == {"detail": "start must be on or before end"} + + +def test_dashboard_binds_filters_in_both_queries(monkeypatch): + calls = [] + + def fake_run(sql, params=None): + calls.append((sql, params)) + if "GROUP BY t" in sql: + result = SimpleNamespace( + result_rows=[ + (datetime(2020, 1, 1), 1.1, 1.2, 1.0, 1.15, 42, 0.0001) + ] + ) + return result, 2.0, 42, 1.5 + + result = SimpleNamespace( + result_rows=[(42, 0.0001, 0.0002, 1.15, 1.0, 1.2)] + ) + return result, 1.0, 42, 0.5 + + monkeypatch.setattr(dashboard_app, "_run", fake_run) + + response = client.get( + "/api/dashboard", + params={ + "pair": "EUR/USD", + "start": "2020-01-01", + "end": "2020-01-02", + "bucket": "hour", + }, + ) + + assert response.status_code == 200 + assert response.json()["timing"] == { + "server_ms": 2.0, + "wall_ms": 3.0, + "rows_read": 84, + "queries": 2, + } + assert len(calls) == 2 + for sql, params in calls: + assert "base = {base:String}" in sql + assert "quote = {quote:String}" in sql + assert params == { + "base": "EUR", + "quote": "USD", + "start": "2020-01-01 00:00:00", + "end": "2020-01-03 00:00:00", + } diff --git a/workshop_public/snowflake_migration_lab/.gitignore b/workshop_public/snowflake_migration_lab/.gitignore new file mode 100644 index 0000000..ea70920 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/.gitignore @@ -0,0 +1,65 @@ +# ── Credentials & secrets ──────────────────────────────────────────────────── +.env +.env.* +!.env.example +*.tfvars +!*.tfvars.example + +# ── Terraform ──────────────────────────────────────────────────────────────── +**/.terraform/ +**/.terraform.lock.hcl +**/terraform.tfstate +**/terraform.tfstate.backup +**/terraform.tfstate.d/ +**/tfplan +# Remote-state backend config: holds the bucket, region, and lock-table names. +# Copy it from backend.hcl.example, which stays tracked. +**/backend.hcl +!**/backend.hcl.example + +# ── Python ─────────────────────────────────────────────────────────────────── +__pycache__/ +*.py[cod] +*.pyo +.venv/ +venv/ +env/ + +# ── dbt ────────────────────────────────────────────────────────────────────── +**/dbt_packages/ +**/dbt_modules/ +**/target/ +**/logs/ +# profiles.yml contains credentials; use profiles.yml.example instead +profiles.yml +# .user.yml is a dbt-generated anonymous telemetry UUID; machine-specific +.user.yml + +# ── Lab-generated outputs ──────────────────────────────────────────────────── +# Auto-generated by profiling script (Part 2) +**/profile_report.md +# Auto-generated by setup.sh (Part 3) +**/.clickhouse_state +# Benchmark CSVs produced by run_benchmark.sh +**/benchmark_results_*.csv + +# ── Partner assessment submissions (keep only blank template) ──────────────── +04-evaluation/assessment_*.md +04-evaluation/docs/answer-key.md + +# ── Superset / Docker volumes ──────────────────────────────────────────────── +**/superset_home/ +**/superset/postgres_data/ +/tmp/superset_import_*.zip +# Dashboard export ZIPs are source assets — keep them despite root *.zip exclusion +!**/superset/dashboards/*.zip + +# ── OS & editor ────────────────────────────────────────────────────────────── +.DS_Store +.DS_Store? +._* +Thumbs.db +.idea/ +.vscode/ +*.swp +*.swo diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/.env.example b/workshop_public/snowflake_migration_lab/01-setup-snowflake/.env.example new file mode 100644 index 0000000..6c519c4 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/.env.example @@ -0,0 +1,34 @@ +# ============================================================ +# .env.example — Environment variables for NYC Taxi lab setup +# Copy to .env and fill in your values. +# NEVER commit .env to source control. +# ============================================================ + +# Snowflake credentials (required) +export SNOWFLAKE_ORG=MYORG # Your Snowflake org name +export SNOWFLAKE_ACCOUNT=MYACCOUNT # Your account name (without org prefix) +export SNOWFLAKE_USER=TERRAFORM_SVC # User with SYSADMIN + SECURITYADMIN privileges +export SNOWFLAKE_PASSWORD= # Set password OR private key path below +# export SNOWFLAKE_PRIVATE_KEY_PATH=~/.ssh/snowflake_rsa_key.p8 + +# Lab metadata +export LAB_ENVIRONMENT=lab +export LAB_COHORT=fy27-q1 + +# Superset +export SUPERSET_SECRET_KEY=change-me-to-random-32char-string-here +export SUPERSET_DB_PASSWORD=superset +export SUPERSET_ADMIN_USER=admin +export SUPERSET_ADMIN_PASSWORD=admin + +# Trip Producer (docker-compose service) +export TRIPS_PER_MINUTE=60 # inserts per minute into TRIPS_RAW (keeps CDC stream + dashboards live) +export BATCH_INTERVAL_SECONDS=10 # how often to flush a batch (TRIPS_PER_MINUTE / 6 rows per flush) +export PRODUCER_ROLE=LOADER_ROLE # Snowflake role used by the producer (INSERT on RAW only) +export PRODUCER_WAREHOUSE=TRANSFORM_WH + +# ClickHouse Cloud (for Act 2 — pre-configure now so Superset has the connection ready) +# export CLICKHOUSE_HOST=your-instance.clickhouse.cloud +# export CLICKHOUSE_PORT=8443 +# export CLICKHOUSE_USER=default +# export CLICKHOUSE_PASSWORD= diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/.gitignore b/workshop_public/snowflake_migration_lab/01-setup-snowflake/.gitignore new file mode 100644 index 0000000..21d0b89 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/.gitignore @@ -0,0 +1 @@ +.venv/ diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/dbt_project.yml b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/dbt_project.yml new file mode 100644 index 0000000..a04baf4 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/dbt_project.yml @@ -0,0 +1,31 @@ +name: 'nyc_taxi_dbt' +version: '1.0.0' +config-version: 2 + +profile: 'nyc_taxi' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" +clean-targets: ["target", "dbt_packages"] + +models: + nyc_taxi_dbt: + staging: + +schema: STAGING + +materialized: view + intermediate: + +schema: STAGING + +materialized: ephemeral + analytics: + +schema: ANALYTICS + +materialized: table + fact_trips: + +materialized: incremental + agg_hourly_zone_trips: + +materialized: incremental diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_schema_name.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_schema_name.sql new file mode 100644 index 0000000..6e2db19 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_schema_name.sql @@ -0,0 +1,10 @@ +-- Override dbt's default schema naming so that models with a custom schema +-- (e.g. +schema: ANALYTICS) land directly in that schema rather than being +-- prefixed with the target schema (e.g. STAGING_ANALYTICS). +{% macro generate_schema_name(custom_schema_name, node) -%} + {%- if custom_schema_name is none -%} + {{ target.schema | upper }} + {%- else -%} + {{ custom_schema_name | upper }} + {%- endif -%} +{%- endmacro %} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_surrogate_key.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_surrogate_key.sql new file mode 100644 index 0000000..f389fdb --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/macros/generate_surrogate_key.sql @@ -0,0 +1,5 @@ +{% macro generate_surrogate_key(field_list) %} + -- Wrapper around dbt_utils.generate_surrogate_key for consistent SK generation + -- Used in models that need a stable surrogate key from natural keys + {{ dbt_utils.generate_surrogate_key(field_list) }} +{% endmacro %} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/agg_hourly_zone_trips.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/agg_hourly_zone_trips.sql new file mode 100644 index 0000000..c21d8cd --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/agg_hourly_zone_trips.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized = 'incremental', + unique_key = ['hour_bucket', 'zone_id'], + incremental_strategy = 'merge', + schema = 'ANALYTICS', + tags = ['hourly', 'aggregate'], + post_hook = "ALTER TABLE {{ this }} CLUSTER BY (hour_bucket)" + ) +}} + +-- Migration note: This MERGE incremental strategy is one of the most challenging +-- translation problems. ClickHouse has no native MERGE. +-- ClickHouse equivalent: ReplacingMergeTree with _version column + FINAL in queries +-- OR: explicit INSERT + DELETE using CollapsingMergeTree + +SELECT + DATE_TRUNC('hour', pickup_at) AS hour_bucket, + pickup_location_id AS zone_id, + COUNT(*) AS trips, + SUM(total_amount_usd) AS revenue, + AVG(trip_distance_miles) AS avg_distance, + CURRENT_TIMESTAMP() AS updated_at + +FROM {{ ref('stg_trips') }} + +{% if is_incremental() %} + WHERE pickup_at >= DATEADD('hour', -2, CURRENT_TIMESTAMP()) +{% endif %} + +GROUP BY 1, 2 diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_date.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_date.sql new file mode 100644 index 0000000..b9b3926 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_date.sql @@ -0,0 +1,90 @@ +{{ + config( + materialized = 'table', + schema = 'ANALYTICS' + ) +}} + +-- Generate a date spine from 2009-01-01 to 2029-12-31 +-- Covers full NYC Taxi dataset history plus forward planning +WITH date_spine AS ( + {{ dbt_utils.date_spine( + datepart = "day", + start_date = "cast('2009-01-01' as date)", + end_date = "cast('2029-12-31' as date)" + ) }} +), + +-- US Federal holidays (static reference, extended annually) +us_holidays AS ( + SELECT holiday_date, holiday_name FROM (VALUES + -- 2019 + ('2019-01-01'::DATE, 'New Year''s Day'), + ('2019-01-21'::DATE, 'MLK Day'), + ('2019-02-18'::DATE, 'Presidents'' Day'), + ('2019-05-27'::DATE, 'Memorial Day'), + ('2019-07-04'::DATE, 'Independence Day'), + ('2019-09-02'::DATE, 'Labor Day'), + ('2019-11-11'::DATE, 'Veterans Day'), + ('2019-11-28'::DATE, 'Thanksgiving'), + ('2019-12-25'::DATE, 'Christmas'), + -- 2020 + ('2020-01-01'::DATE, 'New Year''s Day'), + ('2020-01-20'::DATE, 'MLK Day'), + ('2020-02-17'::DATE, 'Presidents'' Day'), + ('2020-05-25'::DATE, 'Memorial Day'), + ('2020-07-04'::DATE, 'Independence Day'), + ('2020-09-07'::DATE, 'Labor Day'), + ('2020-11-11'::DATE, 'Veterans Day'), + ('2020-11-26'::DATE, 'Thanksgiving'), + ('2020-12-25'::DATE, 'Christmas'), + -- 2021 + ('2021-01-01'::DATE, 'New Year''s Day'), + ('2021-01-18'::DATE, 'MLK Day'), + ('2021-02-15'::DATE, 'Presidents'' Day'), + ('2021-05-31'::DATE, 'Memorial Day'), + ('2021-07-05'::DATE, 'Independence Day (observed)'), + ('2021-09-06'::DATE, 'Labor Day'), + ('2021-11-11'::DATE, 'Veterans Day'), + ('2021-11-25'::DATE, 'Thanksgiving'), + ('2021-12-25'::DATE, 'Christmas'), + -- 2022 + ('2022-01-17'::DATE, 'MLK Day'), + ('2022-02-21'::DATE, 'Presidents'' Day'), + ('2022-05-30'::DATE, 'Memorial Day'), + ('2022-07-04'::DATE, 'Independence Day'), + ('2022-09-05'::DATE, 'Labor Day'), + ('2022-11-11'::DATE, 'Veterans Day'), + ('2022-11-24'::DATE, 'Thanksgiving'), + ('2022-12-26'::DATE, 'Christmas (observed)'), + -- 2023 + ('2023-01-02'::DATE, 'New Year''s Day (observed)'), + ('2023-01-16'::DATE, 'MLK Day'), + ('2023-02-20'::DATE, 'Presidents'' Day'), + ('2023-05-29'::DATE, 'Memorial Day'), + ('2023-07-04'::DATE, 'Independence Day'), + ('2023-09-04'::DATE, 'Labor Day'), + ('2023-11-10'::DATE, 'Veterans Day (observed)'), + ('2023-11-23'::DATE, 'Thanksgiving'), + ('2023-12-25'::DATE, 'Christmas') + ) t (holiday_date, holiday_name) +), + +enriched AS ( + SELECT + date_day::DATE AS date_day, + TO_CHAR(date_day, 'DY') AS day_of_week, + DAYOFWEEK(date_day) AS day_of_week_num, + MONTH(date_day) AS month_num, + TO_CHAR(date_day, 'MON') AS month_name, + QUARTER(date_day) AS quarter_num, + YEAR(date_day) AS year_num, + CONCAT('FY', RIGHT(YEAR(date_day)::VARCHAR, 2), 'Q', QUARTER(date_day)) AS fiscal_quarter, + CASE WHEN DAYOFWEEK(date_day) IN (0, 6) THEN TRUE ELSE FALSE END AS is_weekend, + CASE WHEN h.holiday_date IS NOT NULL THEN TRUE ELSE FALSE END AS is_holiday, + h.holiday_name + FROM date_spine + LEFT JOIN us_holidays h ON date_spine.date_day::DATE = h.holiday_date +) + +SELECT * FROM enriched diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_payment_type.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_payment_type.sql new file mode 100644 index 0000000..1bbacbd --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_payment_type.sql @@ -0,0 +1,10 @@ +{{ + config( + materialized = 'table', + schema = 'ANALYTICS' + ) +}} + +-- Payment type reference — seeded by scripts/01_create_tables.sql +SELECT payment_type_id, payment_code, payment_desc +FROM NYC_TAXI_DB.ANALYTICS.DIM_PAYMENT_TYPE diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_taxi_zones.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_taxi_zones.sql new file mode 100644 index 0000000..cef4ebe --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_taxi_zones.sql @@ -0,0 +1,15 @@ +{{ + config( + materialized = 'table', + schema = 'ANALYTICS' + ) +}} + +-- Passthrough — data seeded directly by scripts/02_seed_data.sql +-- This model provides dbt lineage and enforces column naming conventions +SELECT + location_id, + borough, + zone, + service_zone +FROM {{ ref('stg_taxi_zones') }} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_vendor.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_vendor.sql new file mode 100644 index 0000000..a4bd962 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/dim_vendor.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized = 'table', + schema = 'ANALYTICS' + ) +}} + +SELECT vendor_id, vendor_code, vendor_name +FROM NYC_TAXI_DB.ANALYTICS.DIM_VENDOR diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/fact_trips.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/fact_trips.sql new file mode 100644 index 0000000..9aa4f13 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/fact_trips.sql @@ -0,0 +1,46 @@ +{{ + config( + materialized = 'incremental', + unique_key = 'trip_id', + incremental_strategy = 'merge', + schema = 'ANALYTICS', + cluster_by = ['pickup_at::DATE'], + tags = ['daily', 'core'] + ) +}} + +SELECT + trip_id, + pickup_at, + dropoff_at, + duration_minutes, + trip_distance_miles, + total_amount_usd, + tip_amount_usd, + fare_amount_usd, + extra_amount_usd, + mta_tax_usd, + tolls_amount_usd, + passenger_count, + driver_rating, + vehicle_type, + app_platform, + surge_multiplier, + traffic_level, + pickup_borough, + pickup_zone, + pickup_service_zone, + dropoff_borough, + dropoff_zone, + payment_type, + vendor_name, + pickup_day_of_week, + fiscal_quarter, + is_weekend, + is_holiday, + ingested_at +FROM {{ ref('int_trips_enriched') }} + +{% if is_incremental() %} + WHERE pickup_at > (SELECT MAX(pickup_at) FROM {{ this }}) +{% endif %} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/schema.yml b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/schema.yml new file mode 100644 index 0000000..7fc8150 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/analytics/schema.yml @@ -0,0 +1,39 @@ +version: 2 + +models: + - name: fact_trips + description: "Central fact table — one row per trip, fully denormalized star schema. 50M rows." + columns: + - name: trip_id + tests: + - not_null + - unique + - name: pickup_at + tests: + - not_null + - name: total_amount_usd + tests: + - not_null + - name: payment_type + tests: + - not_null + - name: pickup_borough + description: "Enriched from DIM_TAXI_ZONES join" + + - name: dim_date + description: "Date spine 2009–2029 with fiscal quarters, day-of-week, and US federal holiday flags" + columns: + - name: date_day + tests: + - not_null + - unique + + - name: agg_hourly_zone_trips + description: "Pre-aggregated hourly zone trip metrics. Updated every hour via MERGE. Primary migration challenge: no MERGE in ClickHouse." + columns: + - name: hour_bucket + tests: + - not_null + - name: zone_id + tests: + - not_null diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/intermediate/int_trips_enriched.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/intermediate/int_trips_enriched.sql new file mode 100644 index 0000000..f123de8 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/intermediate/int_trips_enriched.sql @@ -0,0 +1,72 @@ +{{ + config( + materialized = 'ephemeral' + ) +}} + +-- Intermediate model: trips joined to all dimension lookups +-- Ephemeral — compiled inline into downstream models, no physical table + +SELECT + t.trip_id, + t.pickup_at, + t.dropoff_at, + t.duration_minutes, + t.trip_distance_miles, + t.total_amount_usd, + t.tip_amount_usd, + t.fare_amount_usd, + t.extra_amount_usd, + t.mta_tax_usd, + t.tolls_amount_usd, + t.passenger_count, + t.driver_rating, + t.driver_trips_completed, + t.vehicle_type, + t.app_platform, + t.app_version, + t.surge_multiplier, + t.traffic_level, + t.rate_code_id, + t.store_fwd_flag, + t.ingested_at, + -- Pickup zone + pu.borough AS pickup_borough, + pu.zone AS pickup_zone, + pu.service_zone AS pickup_service_zone, + -- Dropoff zone + do_.borough AS dropoff_borough, + do_.zone AS dropoff_zone, + do_.service_zone AS dropoff_service_zone, + -- Payment + pt.payment_code AS payment_code, + pt.payment_desc AS payment_type, + -- Vendor + v.vendor_code AS vendor_code, + v.vendor_name AS vendor_name, + -- Date attributes + d.day_of_week AS pickup_day_of_week, + d.day_of_week_num AS pickup_day_of_week_num, + d.month_name AS pickup_month, + d.quarter_num AS pickup_quarter, + d.fiscal_quarter AS fiscal_quarter, + d.is_weekend AS is_weekend, + d.is_holiday AS is_holiday, + d.holiday_name AS holiday_name + +FROM {{ ref('stg_trips') }} t + +LEFT JOIN {{ ref('stg_taxi_zones') }} pu + ON t.pickup_location_id = pu.location_id + +LEFT JOIN {{ ref('stg_taxi_zones') }} do_ + ON t.dropoff_location_id = do_.location_id + +LEFT JOIN NYC_TAXI_DB.ANALYTICS.DIM_PAYMENT_TYPE pt + ON t.payment_type_id = pt.payment_type_id + +LEFT JOIN NYC_TAXI_DB.ANALYTICS.DIM_VENDOR v + ON t.vendor_id = v.vendor_id + +LEFT JOIN NYC_TAXI_DB.ANALYTICS.DIM_DATE d + ON t.pickup_at::DATE = d.date_day diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/sources.yml b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/sources.yml new file mode 100644 index 0000000..c78bfb0 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/sources.yml @@ -0,0 +1,23 @@ +version: 2 + +sources: + - name: raw + database: NYC_TAXI_DB + schema: RAW + description: "Raw ingested NYC Taxi data — immutable source of truth" + tables: + - name: TRIPS_RAW + description: "All 50M trip records as received from TLC with synthetic JSON metadata" + columns: + - name: TRIP_ID + description: "UUID generated on ingest" + tests: + - not_null + - name: PICKUP_DATETIME + tests: + - not_null + - name: TOTAL_AMOUNT + tests: + - not_null + - name: TRIP_METADATA + description: "VARIANT: driver rating, app version, surge info" diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/schema.yml b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/schema.yml new file mode 100644 index 0000000..f0f8683 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/schema.yml @@ -0,0 +1,44 @@ +version: 2 + +models: + - name: stg_trips + description: "Cleaned and typed staging layer for NYC Taxi trip records. VARIANT column flattened to typed columns." + columns: + - name: trip_id + description: "UUID — primary key" + tests: + - not_null + - unique + - name: pickup_at + tests: + - not_null + - name: dropoff_at + tests: + - not_null + - name: total_amount_usd + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_between: + min_value: 0 + max_value: 1000 + - name: duration_minutes + tests: + - dbt_expectations.expect_column_values_to_be_between: + min_value: 1 + max_value: 600 + - name: payment_type_id + tests: + - accepted_values: + values: [1, 2, 3, 4, 5, 6] + - name: driver_rating + description: "Extracted from TRIP_METADATA VARIANT — null if metadata absent" + - name: surge_multiplier + description: "App surge pricing factor from VARIANT. 1.0 = no surge." + + - name: stg_taxi_zones + description: "Cleaned taxi zone dimension from TLC lookup" + columns: + - name: location_id + tests: + - not_null + - unique diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_taxi_zones.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_taxi_zones.sql new file mode 100644 index 0000000..328a96b --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_taxi_zones.sql @@ -0,0 +1,16 @@ +{{ + config( + materialized = 'view', + schema = 'STAGING' + ) +}} + +-- Zone data is seeded directly into ANALYTICS.DIM_TAXI_ZONES via scripts/02_seed_data.sql +-- This staging view adds light cleaning and serves as the dbt lineage node +SELECT + LOCATION_ID AS location_id, + COALESCE(BOROUGH, 'Unknown') AS borough, + COALESCE(ZONE, 'Unknown') AS zone, + COALESCE(SERVICE_ZONE, 'Unknown') AS service_zone +FROM NYC_TAXI_DB.ANALYTICS.DIM_TAXI_ZONES +WHERE LOCATION_ID IS NOT NULL diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_trips.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_trips.sql new file mode 100644 index 0000000..4220e70 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/models/staging/stg_trips.sql @@ -0,0 +1,51 @@ +{{ + config( + materialized = 'view', + schema = 'STAGING' + ) +}} + +WITH source AS ( + SELECT * FROM {{ source('raw', 'TRIPS_RAW') }} +), + +flattened AS ( + SELECT + TRIP_ID AS trip_id, + VENDOR_ID AS vendor_id, + PICKUP_DATETIME AS pickup_at, + DROPOFF_DATETIME AS dropoff_at, + DATEDIFF('minute', PICKUP_DATETIME, DROPOFF_DATETIME) AS duration_minutes, + PASSENGER_COUNT AS passenger_count, + TRIP_DISTANCE AS trip_distance_miles, + TOTAL_AMOUNT AS total_amount_usd, + TIP_AMOUNT AS tip_amount_usd, + FARE_AMOUNT AS fare_amount_usd, + EXTRA AS extra_amount_usd, + MTA_TAX AS mta_tax_usd, + TOLLS_AMOUNT AS tolls_amount_usd, + PU_LOCATION_ID AS pickup_location_id, + DO_LOCATION_ID AS dropoff_location_id, + PAYMENT_TYPE AS payment_type_id, + RATECODE_ID AS rate_code_id, + STORE_FWD_FLAG AS store_fwd_flag, + INGESTED_AT AS ingested_at, + -- VARIANT column flattened to typed columns + -- Migration note: Snowflake colon-path syntax → ClickHouse JSONExtractFloat/String + TRIP_METADATA:driver.rating::FLOAT AS driver_rating, + TRIP_METADATA:driver.trips_completed::INTEGER AS driver_trips_completed, + TRIP_METADATA:driver.vehicle_type::VARCHAR AS vehicle_type, + TRIP_METADATA:app.platform::VARCHAR AS app_platform, + TRIP_METADATA:app.version::VARCHAR AS app_version, + TRIP_METADATA:app.surge_multiplier::FLOAT AS surge_multiplier, + TRIP_METADATA:route.estimated_minutes::INTEGER AS route_estimated_minutes, + TRIP_METADATA:route.actual_minutes::INTEGER AS route_actual_minutes, + TRIP_METADATA:route.traffic_level::VARCHAR AS traffic_level + FROM source + WHERE TRIP_ID IS NOT NULL + AND PICKUP_DATETIME IS NOT NULL + AND DROPOFF_DATETIME IS NOT NULL + AND DROPOFF_DATETIME > PICKUP_DATETIME -- exclude negative-duration trips +) + +SELECT * FROM flattened diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/package-lock.yml b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/package-lock.yml new file mode 100644 index 0000000..499974e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/package-lock.yml @@ -0,0 +1,11 @@ +packages: + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.3.3 + - name: dbt_expectations + package: calogica/dbt_expectations + version: 0.10.4 + - name: dbt_date + package: calogica/dbt_date + version: 0.10.1 +sha1_hash: db4d84e4edc277bea424e7201c2019491576db2d diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/packages.yml b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/packages.yml new file mode 100644 index 0000000..e086e84 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/packages.yml @@ -0,0 +1,5 @@ +packages: + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<2.0.0"] + - package: calogica/dbt_expectations + version: [">=0.10.0", "<1.0.0"] diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/profiles.yml.example b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/profiles.yml.example new file mode 100644 index 0000000..0e4db25 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/profiles.yml.example @@ -0,0 +1,29 @@ +# Copy to ~/.dbt/profiles.yml or set DBT_PROFILES_DIR=. +# NEVER commit with real credentials. +nyc_taxi: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ORG') }}-{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('SNOWFLAKE_PASSWORD') }}" + # OR use key-pair: + # private_key_path: "{{ env_var('SNOWFLAKE_PRIVATE_KEY_PATH') }}" + role: DBT_ROLE + database: NYC_TAXI_DB + warehouse: TRANSFORM_WH + schema: STAGING + threads: 4 + client_session_keep_alive: false + prod: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + private_key_path: "{{ env_var('SNOWFLAKE_PRIVATE_KEY_PATH') }}" + role: DBT_ROLE + database: NYC_TAXI_DB + warehouse: TRANSFORM_WH + schema: STAGING + threads: 8 + client_session_keep_alive: false diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/snapshots/driver_rating_snapshot.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/snapshots/driver_rating_snapshot.sql new file mode 100644 index 0000000..8792847 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/snapshots/driver_rating_snapshot.sql @@ -0,0 +1,25 @@ +{% snapshot driver_rating_snapshot %} + +{{ + config( + target_schema = 'STAGING', + unique_key = 'trip_id', + strategy = 'check', + check_cols = ['driver_rating', 'vehicle_type'], + invalidate_hard_deletes = True + ) +}} + +-- SCD Type 2: tracks changes to driver rating and vehicle type over time +-- Demonstrates how Snowflake snapshot patterns translate to ClickHouse +-- ClickHouse equivalent: ReplacingMergeTree with version column +SELECT + trip_id, + driver_rating, + vehicle_type, + app_platform, + CURRENT_TIMESTAMP() AS snapshot_at +FROM {{ ref('stg_trips') }} +WHERE driver_rating IS NOT NULL + +{% endsnapshot %} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_revenue_positive.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_revenue_positive.sql new file mode 100644 index 0000000..d06c09d --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_revenue_positive.sql @@ -0,0 +1,9 @@ +-- Custom test: all fare amounts must be non-negative +-- A passing test returns 0 rows +SELECT + trip_id, + total_amount_usd, + fare_amount_usd +FROM {{ ref('fact_trips') }} +WHERE total_amount_usd < 0 + OR fare_amount_usd < 0 diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_trips_not_future.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_trips_not_future.sql new file mode 100644 index 0000000..12d29c4 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/dbt/nyc_taxi_dbt/tests/assert_trips_not_future.sql @@ -0,0 +1,8 @@ +-- Custom test: no future-dated pickups +-- A passing test returns 0 rows +SELECT + trip_id, + pickup_at, + CURRENT_TIMESTAMP() AS now +FROM {{ ref('fact_trips') }} +WHERE pickup_at > CURRENT_TIMESTAMP() diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/Dockerfile b/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/Dockerfile new file mode 100644 index 0000000..275d835 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install dependencies first (layer-cached separately from source) +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY producer.py . + +# Unbuffered output so logs appear immediately in docker logs +ENV PYTHONUNBUFFERED=1 + +CMD ["python", "-u", "producer.py"] diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/producer.py b/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/producer.py new file mode 100644 index 0000000..0237d4c --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/producer.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +""" +NYC Taxi Trip Producer +---------------------- +Continuously generates realistic fake taxi trips and inserts them into +NYC_TAXI_DB.RAW.TRIPS_RAW, keeping the CDC stream and dashboards live. + +Configuration (env vars): + SNOWFLAKE_ACCOUNT ORG-ACCOUNT format (required) + SNOWFLAKE_USER Snowflake username (required) + SNOWFLAKE_PASSWORD Password (required unless using key-pair) + SNOWFLAKE_PRIVATE_KEY_PATH Path to .p8 file (alternative to password) + SNOWFLAKE_ROLE Default: LOADER_ROLE + SNOWFLAKE_WAREHOUSE Default: TRANSFORM_WH + TRIPS_PER_MINUTE Default: 60 + BATCH_INTERVAL_SECONDS Default: 10 + LOG_LEVEL Default: INFO +""" +import json +import logging +import os +import random +import signal +import sys +import time +import uuid +from datetime import datetime, timezone, timedelta + +import snowflake.connector + +# ── Configuration ────────────────────────────────────────────────────────────── + +SNOWFLAKE_ACCOUNT = os.environ["SNOWFLAKE_ACCOUNT"] +SNOWFLAKE_USER = os.environ["SNOWFLAKE_USER"] +SNOWFLAKE_PASSWORD = os.environ.get("SNOWFLAKE_PASSWORD", "") +SNOWFLAKE_PRIVATE_KEY = os.environ.get("SNOWFLAKE_PRIVATE_KEY_PATH", "") +SNOWFLAKE_ROLE = os.environ.get("SNOWFLAKE_ROLE", "LOADER_ROLE") +SNOWFLAKE_WAREHOUSE = os.environ.get("SNOWFLAKE_WAREHOUSE", "TRANSFORM_WH") +SNOWFLAKE_DATABASE = "NYC_TAXI_DB" +SNOWFLAKE_SCHEMA = "RAW" + +TRIPS_PER_MINUTE = float(os.environ.get("TRIPS_PER_MINUTE", "60")) +BATCH_INTERVAL_SECS = int(os.environ.get("BATCH_INTERVAL_SECONDS", "10")) +TRIPS_PER_BATCH = max(1, round(TRIPS_PER_MINUTE * BATCH_INTERVAL_SECS / 60)) + +logging.basicConfig( + level=getattr(logging, os.environ.get("LOG_LEVEL", "INFO").upper(), logging.INFO), + format="%(asctime)s [producer] %(levelname)s %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, +) +log = logging.getLogger(__name__) + +# ── Static reference data ────────────────────────────────────────────────────── + +# Manhattan zones (roughly 1-103 + some higher IDs) appear 4x more often +_MANHATTAN = ( + list(range(4, 12)) + list(range(13, 45)) + list(range(46, 78)) + + list(range(79, 104)) + [107, 113, 114, 125, 140, 141, 142, 143, 144, 148, + 151, 152, 153, 158, 161, 162, 163, 164, 166, 170, 186, 194, 202, 209, + 211, 224, 231, 234, 236, 239, 243, 244, 246, 249, 261, 262, 263] +) +_OTHER = [z for z in range(1, 266) if z not in _MANHATTAN] +WEIGHTED_ZONES = _MANHATTAN * 4 + _OTHER + +VEHICLE_TYPES = ["Sedan", "SUV", "Minivan", "Luxury"] +PLATFORMS = ["iOS", "Android", "Web"] +TRAFFIC_LEVELS = ["none", "light", "moderate", "heavy"] +TRAFFIC_WEIGHTS = [20, 40, 30, 10] + +# Realistic app version pool +APP_VERSIONS = [ + f"{maj}.{minor}.{patch}" + for maj in range(2, 5) + for minor in range(0, 8) + for patch in range(0, 5) +] + +# ── Trip generation ──────────────────────────────────────────────────────────── + +def make_trip() -> tuple: + """Return one row tuple matching TRIPS_RAW column order.""" + vendor_id = random.randint(1, 3) + pu_zone = random.choice(WEIGHTED_ZONES) + do_zone = random.choice(WEIGHTED_ZONES) + passenger_count = random.choices([1, 2, 3, 4, 5, 6], weights=[55, 20, 10, 8, 5, 2])[0] + payment_type = random.choices([1, 2, 3, 4, 5, 6], weights=[65, 30, 1, 2, 1, 1])[0] + rate_code_id = random.choices([1, 2, 3, 4, 5, 6], weights=[88, 4, 1, 1, 4, 2])[0] + store_fwd = random.choices(["N", "Y"], weights=[98, 2])[0] + + # Distance: exponential distribution gives realistic long-tail (mean ~4 miles) + distance = round(max(0.1, min(random.expovariate(1 / 4.0), 60.0)), 2) + + # Duration roughly proportional to distance + traffic noise + traffic_level = random.choices(TRAFFIC_LEVELS, weights=TRAFFIC_WEIGHTS)[0] + traffic_factor = {"none": 3.5, "light": 4.5, "moderate": 6.0, "heavy": 8.5}[traffic_level] + duration_min = max(3, min(120, int(distance * traffic_factor + random.gauss(3, 2)))) + estimated_min = max(3, int(distance * 4.5 + random.gauss(2, 1))) + + # Trip ended within the last 90 seconds (just completed) + dropoff_dt = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(seconds=random.randint(0, 90)) + pickup_dt = dropoff_dt - timedelta(minutes=duration_min) + + # Fare: TLC rate card approximation + # Standard rate: $3.00 base + $1.75/mile + fare = round(3.00 + distance * 1.75, 2) + extra = round(random.choices([0.0, 0.5, 1.0], weights=[60, 30, 10])[0], 2) + mta = 0.50 + tolls = round(random.choices([0.0, 6.12, 11.52], weights=[85, 10, 5])[0], 2) + tip = round(fare * random.uniform(0.15, 0.25), 2) if payment_type == 1 else 0.0 + total = round(fare + extra + mta + tip + tolls, 2) + + # Surge: rare high surge, occasional medium, common none + surge_roll = random.random() + if surge_roll < 0.03: + surge = round(random.uniform(2.0, 3.5), 1) + elif surge_roll < 0.10: + surge = round(random.uniform(1.5, 2.0), 1) + elif surge_roll < 0.25: + surge = round(random.uniform(1.1, 1.5), 1) + else: + surge = 1.0 + + metadata = json.dumps({ + "driver": { + "rating": max(1.0, min(5.0, round(random.gauss(4.6, 0.3), 1))), + "trips_completed": random.randint(50, 8000), + "vehicle_type": random.choice(VEHICLE_TYPES), + }, + "app": { + "version": random.choice(APP_VERSIONS), + "platform": random.choice(PLATFORMS), + "surge_multiplier": surge, + }, + "route": { + "estimated_minutes": estimated_min, + "actual_minutes": duration_min, + "traffic_level": traffic_level, + }, + }) + + return ( + str(uuid.uuid4()), # TRIP_ID + vendor_id, # VENDOR_ID + pickup_dt, # PICKUP_DATETIME + dropoff_dt, # DROPOFF_DATETIME + passenger_count, # PASSENGER_COUNT + distance, # TRIP_DISTANCE + rate_code_id, # RATECODE_ID + store_fwd, # STORE_FWD_FLAG + pu_zone, # PU_LOCATION_ID + do_zone, # DO_LOCATION_ID + payment_type, # PAYMENT_TYPE + fare, # FARE_AMOUNT + extra, # EXTRA + mta, # MTA_TAX + tip, # TIP_AMOUNT + tolls, # TOLLS_AMOUNT + total, # TOTAL_AMOUNT + metadata, # TRIP_METADATA → PARSE_JSON + ) + + +INSERT_SQL = """ +INSERT INTO NYC_TAXI_DB.RAW.TRIPS_RAW ( + TRIP_ID, VENDOR_ID, PICKUP_DATETIME, DROPOFF_DATETIME, + PASSENGER_COUNT, TRIP_DISTANCE, RATECODE_ID, STORE_FWD_FLAG, + PU_LOCATION_ID, DO_LOCATION_ID, PAYMENT_TYPE, + FARE_AMOUNT, EXTRA, MTA_TAX, TIP_AMOUNT, TOLLS_AMOUNT, TOTAL_AMOUNT, + TRIP_METADATA +) +SELECT + %s, %s, %s, %s, + %s, %s, %s, %s, + %s, %s, %s, + %s, %s, %s, %s, %s, %s, + PARSE_JSON(%s) +""" + +# ── Snowflake connection ─────────────────────────────────────────────────────── + +def _load_private_key(path: str): + """Load an unencrypted RSA private key (.p8) for key-pair auth.""" + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.serialization import ( + Encoding, PrivateFormat, NoEncryption, load_pem_private_key + ) + with open(path, "rb") as f: + private_key = load_pem_private_key(f.read(), password=None, backend=default_backend()) + return private_key.private_bytes( + encoding=Encoding.DER, + format=PrivateFormat.PKCS8, + encryption_algorithm=NoEncryption(), + ) + + +def connect() -> snowflake.connector.SnowflakeConnection: + log.info("Connecting account=%s user=%s role=%s warehouse=%s", + SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_ROLE, SNOWFLAKE_WAREHOUSE) + + kwargs = dict( + account = SNOWFLAKE_ACCOUNT, + user = SNOWFLAKE_USER, + role = SNOWFLAKE_ROLE, + warehouse = SNOWFLAKE_WAREHOUSE, + database = SNOWFLAKE_DATABASE, + schema = SNOWFLAKE_SCHEMA, + ) + if SNOWFLAKE_PRIVATE_KEY: + kwargs["private_key"] = _load_private_key(SNOWFLAKE_PRIVATE_KEY) + else: + kwargs["password"] = SNOWFLAKE_PASSWORD + + conn = snowflake.connector.connect(**kwargs) + log.info("Connected.") + return conn + +# ── Main loop ────────────────────────────────────────────────────────────────── + +def run(): + conn = connect() + cursor = conn.cursor() + total = 0 + start = time.monotonic() + + log.info("Producer running — %.0f trips/min, %d per batch, interval %ds", + TRIPS_PER_MINUTE, TRIPS_PER_BATCH, BATCH_INTERVAL_SECS) + + while True: + batch_start = time.monotonic() + rows = [make_trip() for _ in range(TRIPS_PER_BATCH)] + + try: + for row in rows: + cursor.execute(INSERT_SQL, row) + total += len(rows) + elapsed_min = (time.monotonic() - start) / 60 or 0.001 + log.info("✓ inserted %3d trips | total=%6d | actual rate=%.1f trips/min", + len(rows), total, total / elapsed_min) + except snowflake.connector.errors.DatabaseError as exc: + log.error("Insert failed: %s — reconnecting in 5s", exc) + try: + conn.close() + except Exception: + pass + time.sleep(5) + conn = connect() + cursor = conn.cursor() + continue + + sleep_secs = max(0.0, BATCH_INTERVAL_SECS - (time.monotonic() - batch_start)) + time.sleep(sleep_secs) + + +def _shutdown(sig, _frame): + log.info("Received signal %s — shutting down.", sig) + sys.exit(0) + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, _shutdown) + signal.signal(signal.SIGINT, _shutdown) + + log.info("NYC Taxi Trip Producer starting " + "(TRIPS_PER_MINUTE=%.0f BATCH_INTERVAL=%ds TRIPS_PER_BATCH=%d)", + TRIPS_PER_MINUTE, BATCH_INTERVAL_SECS, TRIPS_PER_BATCH) + + # Outer retry loop keeps the container alive through transient Snowflake errors + backoff = 15 + while True: + try: + run() + except KeyboardInterrupt: + log.info("Interrupted.") + sys.exit(0) + except Exception as exc: + log.error("Unexpected error: %s — retrying in %ds", exc, backoff) + time.sleep(backoff) + backoff = min(backoff * 2, 120) # cap at 2 minutes + else: + backoff = 15 # reset on clean restart diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/requirements.txt b/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/requirements.txt new file mode 100644 index 0000000..8401386 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/producer/requirements.txt @@ -0,0 +1,2 @@ +snowflake-connector-python==3.7.0 +cryptography>=41.0.0 # for key-pair auth support diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q1_hourly_revenue_by_borough.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q1_hourly_revenue_by_borough.sql new file mode 100644 index 0000000..b405766 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q1_hourly_revenue_by_borough.sql @@ -0,0 +1,35 @@ +-- Q1: Hourly Revenue by Borough +-- Used by: Operations dashboard, runs every 15 minutes +-- Migration note: DATE_TRUNC is supported in ClickHouse with same syntax +-- NULLIF works in ClickHouse too +-- DATEADD → use pickup_at >= now() - INTERVAL 7 DAY in ClickHouse + +USE WAREHOUSE ANALYTICS_WH; +USE DATABASE NYC_TAXI_DB; + +SELECT + DATE_TRUNC('hour', pickup_at) AS hour_bucket, + pickup_borough, + COUNT(*) AS trip_count, + SUM(total_amount_usd) AS total_revenue, + AVG(tip_amount_usd / NULLIF(fare_amount_usd, 0)) AS avg_tip_rate, + AVG(trip_distance_miles) AS avg_distance_miles +FROM ANALYTICS.FACT_TRIPS +WHERE pickup_at >= DATEADD('day', -7, CURRENT_TIMESTAMP()) + AND pickup_borough IS NOT NULL +GROUP BY 1, 2 +ORDER BY 1 DESC, total_revenue DESC; + +-- ClickHouse equivalent: +-- SELECT +-- toStartOfHour(pickup_at) AS hour_bucket, +-- pickup_borough, +-- count() AS trip_count, +-- sum(total_amount_usd) AS total_revenue, +-- avg(tip_amount_usd / nullIf(fare_amount_usd, 0)) AS avg_tip_rate, +-- avg(trip_distance_miles) AS avg_distance_miles +-- FROM analytics.fact_trips +-- WHERE pickup_at >= now() - INTERVAL 7 DAY +-- AND pickup_borough != '' +-- GROUP BY hour_bucket, pickup_borough +-- ORDER BY hour_bucket DESC, total_revenue DESC; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q2_rolling_7day_avg_distance.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q2_rolling_7day_avg_distance.sql new file mode 100644 index 0000000..9fef3fe --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q2_rolling_7day_avg_distance.sql @@ -0,0 +1,39 @@ +-- Q2: Rolling 7-Day Average Trip Distance +-- Used by: Executive weekly report +-- Migration note: Window frame syntax (ROWS BETWEEN) is nearly identical in ClickHouse +-- Nested aggregate window function (AVG(AVG(...))) works in ClickHouse too + +USE WAREHOUSE ANALYTICS_WH; +USE DATABASE NYC_TAXI_DB; + +SELECT + pickup_at::DATE AS trip_date, + COUNT(*) AS daily_trip_count, + AVG(trip_distance_miles) AS daily_avg_distance, + AVG(AVG(trip_distance_miles)) OVER ( + ORDER BY pickup_at::DATE + ROWS BETWEEN 6 PRECEDING AND CURRENT ROW + ) AS rolling_7d_avg_distance, + SUM(total_amount_usd) AS daily_revenue, + SUM(SUM(total_amount_usd)) OVER ( + ORDER BY pickup_at::DATE + ROWS BETWEEN 6 PRECEDING AND CURRENT ROW + ) AS rolling_7d_revenue +FROM ANALYTICS.FACT_TRIPS +GROUP BY 1 +ORDER BY 1 DESC +LIMIT 365; + +-- ClickHouse equivalent (syntax nearly identical): +-- SELECT +-- toDate(pickup_at) AS trip_date, +-- count() AS daily_trip_count, +-- avg(trip_distance_miles) AS daily_avg_distance, +-- avg(avg(trip_distance_miles)) OVER ( +-- ORDER BY trip_date +-- ROWS BETWEEN 6 PRECEDING AND CURRENT ROW +-- ) AS rolling_7d_avg_distance +-- FROM analytics.fact_trips +-- GROUP BY trip_date +-- ORDER BY trip_date DESC +-- LIMIT 365; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q3_top10_trips_qualify.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q3_top10_trips_qualify.sql new file mode 100644 index 0000000..9a303bf --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q3_top10_trips_qualify.sql @@ -0,0 +1,38 @@ +-- Q3: Top 10 Trips per Borough using QUALIFY +-- Migration note: For this lab, QUALIFY is treated as a dialect gap requiring a subquery rewrite. +-- (ClickHouse does support QUALIFY, but the subquery pattern is portable across all SQL engines. https://clickhouse.com/docs/sql-reference/statements/select/qualify) + +USE WAREHOUSE ANALYTICS_WH; +USE DATABASE NYC_TAXI_DB; + +-- Snowflake version: QUALIFY filters window function inline +SELECT + trip_id, + pickup_at, + pickup_borough, + total_amount_usd, + tip_amount_usd, + trip_distance_miles, + ROW_NUMBER() OVER ( + PARTITION BY pickup_borough + ORDER BY total_amount_usd DESC + ) AS rank_in_borough +FROM ANALYTICS.FACT_TRIPS +WHERE pickup_at::DATE = CURRENT_DATE() - 1 +QUALIFY rank_in_borough <= 10 +ORDER BY pickup_borough, rank_in_borough; + +-- ClickHouse equivalent: subquery rewrite (portable across all SQL engines) +-- SELECT trip_id, pickup_at, pickup_borough, total_amount_usd, tip_amount_usd, trip_distance_miles, rn AS rank_in_borough +-- FROM ( +-- SELECT +-- trip_id, pickup_at, pickup_borough, total_amount_usd, tip_amount_usd, trip_distance_miles, +-- row_number() OVER ( +-- PARTITION BY pickup_borough +-- ORDER BY total_amount_usd DESC +-- ) AS rn +-- FROM analytics.fact_trips +-- WHERE toDate(pickup_at) = today() - 1 +-- ) +-- WHERE rn <= 10 +-- ORDER BY pickup_borough, rn; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q4_driver_rating_lateral_flatten.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q4_driver_rating_lateral_flatten.sql new file mode 100644 index 0000000..44130a1 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q4_driver_rating_lateral_flatten.sql @@ -0,0 +1,37 @@ +-- Q4: Driver Rating Distribution from VARIANT Column +-- Migration note: LATERAL FLATTEN has no direct equivalent in ClickHouse. +-- Options: (1) use JSONExtract functions on the raw JSON column +-- (2) pre-flatten during migration (preferred for performance) + +USE WAREHOUSE ANALYTICS_WH; +USE DATABASE NYC_TAXI_DB; + +-- Snowflake version: LATERAL FLATTEN on VARIANT +SELECT + ROUND(TRIP_METADATA:driver.rating::FLOAT, 1) AS rating_bucket, + COUNT(*) AS trip_count, + AVG(TOTAL_AMOUNT) AS avg_fare, + AVG(DATEDIFF('minute', PICKUP_DATETIME, DROPOFF_DATETIME)) AS avg_duration_minutes +FROM RAW.TRIPS_RAW +WHERE TRIP_METADATA:driver IS NOT NULL + AND TRIP_METADATA:driver.rating IS NOT NULL +GROUP BY 1 +ORDER BY 1; + +-- ClickHouse equivalent (using JSONExtractFloat on String/JSON column): +-- SELECT +-- round(JSONExtractFloat(trip_metadata, 'driver', 'rating'), 1) AS rating_bucket, +-- count() AS trip_count, +-- avg(total_amount) AS avg_fare, +-- avg(dateDiff('minute', pickup_datetime, dropoff_datetime)) AS avg_duration_minutes +-- FROM raw.trips_raw +-- WHERE JSONHas(trip_metadata, 'driver') +-- AND JSONExtractFloat(trip_metadata, 'driver', 'rating') > 0 +-- GROUP BY rating_bucket +-- ORDER BY rating_bucket; +-- +-- OR if pre-flattened into typed columns (recommended): +-- SELECT round(driver_rating, 1) AS rating_bucket, count(), avg(total_amount_usd), avg(duration_minutes) +-- FROM analytics.fact_trips +-- WHERE driver_rating > 0 +-- GROUP BY rating_bucket ORDER BY rating_bucket; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q5_surge_pricing_variant.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q5_surge_pricing_variant.sql new file mode 100644 index 0000000..43ef3f8 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q5_surge_pricing_variant.sql @@ -0,0 +1,39 @@ +-- Q5: Surge Pricing Impact Analysis +-- Migration note: Colon-path VARIANT access → JSONExtractFloat in ClickHouse +-- CASE/WHEN logic is identical + +USE WAREHOUSE ANALYTICS_WH; +USE DATABASE NYC_TAXI_DB; + +-- Snowflake version: colon-path VARIANT access +SELECT + CASE + WHEN TRIP_METADATA:app.surge_multiplier::FLOAT >= 2.0 THEN 'High Surge (2x+)' + WHEN TRIP_METADATA:app.surge_multiplier::FLOAT >= 1.5 THEN 'Medium Surge (1.5–2x)' + WHEN TRIP_METADATA:app.surge_multiplier::FLOAT > 1.0 THEN 'Low Surge (1–1.5x)' + ELSE 'No Surge (1x)' + END AS surge_category, + COUNT(*) AS trip_count, + ROUND(AVG(TOTAL_AMOUNT), 2) AS avg_total_fare, + ROUND(AVG(FARE_AMOUNT), 2) AS avg_base_fare, + ROUND(AVG(PASSENGER_COUNT), 1) AS avg_passengers, + ROUND(AVG(TRIP_DISTANCE), 2) AS avg_distance_miles +FROM RAW.TRIPS_RAW +WHERE TRIP_METADATA:app.surge_multiplier IS NOT NULL +GROUP BY 1 +ORDER BY AVG(TRIP_METADATA:app.surge_multiplier::FLOAT) DESC; + +-- ClickHouse equivalent: +-- SELECT +-- CASE +-- WHEN JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier') >= 2.0 THEN 'High Surge (2x+)' +-- WHEN JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier') >= 1.5 THEN 'Medium Surge (1.5–2x)' +-- WHEN JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier') > 1.0 THEN 'Low Surge (1–1.5x)' +-- ELSE 'No Surge (1x)' +-- END AS surge_category, +-- count() AS trip_count, +-- round(avg(total_amount), 2) AS avg_total_fare +-- FROM raw.trips_raw +-- WHERE JSONHas(trip_metadata, 'app') +-- GROUP BY surge_category +-- ORDER BY avg(JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier')) DESC; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q6_merge_hourly_aggregation.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q6_merge_hourly_aggregation.sql new file mode 100644 index 0000000..b567906 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q6_merge_hourly_aggregation.sql @@ -0,0 +1,63 @@ +-- Q6: Incremental Hourly Zone Aggregation (MERGE) +-- Run by dbt every hour via Snowflake Task +-- Migration note: No MERGE in ClickHouse. Options: +-- (1) ReplacingMergeTree — engine handles dedup on OPTIMIZE or with FINAL +-- (2) INSERT + DELETE pattern with AggregatingMergeTree +-- (3) CollapsingMergeTree for explicit sign-based cancellation + +USE WAREHOUSE TRANSFORM_WH; +USE DATABASE NYC_TAXI_DB; + +-- Snowflake MERGE (Snowflake-specific) +MERGE INTO ANALYTICS.AGG_HOURLY_ZONE_TRIPS AS target +USING ( + SELECT + DATE_TRUNC('hour', pickup_at) AS hour_bucket, + pickup_location_id AS zone_id, + COUNT(*) AS trips, + SUM(total_amount_usd) AS revenue, + AVG(trip_distance_miles) AS avg_distance + FROM ANALYTICS.FACT_TRIPS + WHERE pickup_at >= DATEADD('hour', -2, CURRENT_TIMESTAMP()) + GROUP BY 1, 2 +) AS source +ON target.hour_bucket = source.hour_bucket +AND target.zone_id = source.zone_id +WHEN MATCHED THEN UPDATE SET + target.trips = source.trips, + target.revenue = source.revenue, + target.avg_distance = source.avg_distance, + target.updated_at = CURRENT_TIMESTAMP() +WHEN NOT MATCHED THEN INSERT + (hour_bucket, zone_id, trips, revenue, avg_distance) +VALUES + (source.hour_bucket, source.zone_id, source.trips, + source.revenue, source.avg_distance); + +-- ClickHouse equivalent using ReplacingMergeTree: +-- Table DDL: +-- CREATE TABLE analytics.agg_hourly_zone_trips ( +-- hour_bucket DateTime, +-- zone_id UInt32, +-- trips UInt64, +-- revenue Float64, +-- avg_distance Float64, +-- updated_at DateTime DEFAULT now(), +-- _version UInt64 DEFAULT toUnixTimestamp(now()) +-- ) ENGINE = ReplacingMergeTree(_version) +-- ORDER BY (hour_bucket, zone_id); +-- +-- Insert/upsert: +-- INSERT INTO analytics.agg_hourly_zone_trips +-- SELECT +-- toStartOfHour(pickup_at), pickup_location_id, +-- count(), sum(total_amount_usd), avg(trip_distance_miles), +-- now(), toUnixTimestamp(now()) +-- FROM analytics.fact_trips +-- WHERE pickup_at >= now() - INTERVAL 2 HOUR +-- GROUP BY 1, 2; +-- +-- Query with dedup (FINAL forces merge): +-- SELECT hour_bucket, zone_id, trips, revenue +-- FROM analytics.agg_hourly_zone_trips FINAL +-- ORDER BY hour_bucket DESC; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q7_cdc_stream_consumption.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q7_cdc_stream_consumption.sql new file mode 100644 index 0000000..458d205 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/queries/q7_cdc_stream_consumption.sql @@ -0,0 +1,43 @@ +-- Q7: Consume CDC Stream for Incremental ClickHouse Sync +-- Source-side mechanism for live CDC migration track +-- In the migration lab, this stream is retired at cutover — live writes go directly +-- to ClickHouse via the post-cutover producer (scripts/03_cutover.sh). +-- Migration note: Snowflake Streams have no native ClickHouse equivalent. +-- ClickHouse equivalent: direct producer writes post-cutover, +-- or Debezium → Kafka → ClickHouse for real CDC. + +USE WAREHOUSE TRANSFORM_WH; +USE DATABASE NYC_TAXI_DB; + +-- Read all pending changes from the CDC stream +-- METADATA$ columns are Snowflake-specific stream metadata fields +SELECT + METADATA$ACTION AS cdc_action, -- 'INSERT' or 'DELETE' + METADATA$ISUPDATE AS is_update, -- TRUE for UPDATE events (shown as DELETE+INSERT pair) + METADATA$ROW_ID AS row_id, + TRIP_ID, + VENDOR_ID, + PICKUP_DATETIME, + DROPOFF_DATETIME, + PASSENGER_COUNT, + TRIP_DISTANCE, + PU_LOCATION_ID, + DO_LOCATION_ID, + PAYMENT_TYPE, + FARE_AMOUNT, + TOTAL_AMOUNT, + TIP_AMOUNT, + TRIP_METADATA +FROM NYC_TAXI_DB.RAW.TRIPS_CDC_STREAM +WHERE METADATA$ACTION = 'INSERT' +ORDER BY PICKUP_DATETIME DESC +LIMIT 10000; + +-- To process the full change feed for ClickHouse sync: +-- 1. SELECT all rows from stream (this consumes the stream) +-- 2. INSERT rows where METADATA$ACTION = 'INSERT' into ClickHouse via the producer +-- 3. For UPDATE events: rows come as DELETE + INSERT pair; handle in ClickHouse +-- using ReplacingMergeTree or by processing sign column + +-- Show current stream lag (useful for monitoring sync health) +SELECT SYSTEM$STREAM_BACKLOG_SIZE('NYC_TAXI_DB.RAW.TRIPS_CDC_STREAM') AS stream_backlog_bytes; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/01_create_tables.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/01_create_tables.sql new file mode 100644 index 0000000..9c4582c --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/01_create_tables.sql @@ -0,0 +1,115 @@ +-- ============================================================ +-- Script 01: Create all tables in NYC_TAXI_DB +-- Run after: terraform apply +-- Run before: 02_seed_data.sql +-- ============================================================ + +USE WAREHOUSE TRANSFORM_WH; +USE DATABASE NYC_TAXI_DB; + +-- ============================================================ +-- RAW Layer +-- ============================================================ + +CREATE TABLE IF NOT EXISTS RAW.TRIPS_RAW ( + TRIP_ID VARCHAR(36) NOT NULL, -- UUID generated on ingest + VENDOR_ID INTEGER, + PICKUP_DATETIME TIMESTAMP_NTZ NOT NULL, + DROPOFF_DATETIME TIMESTAMP_NTZ NOT NULL, + PASSENGER_COUNT INTEGER, + TRIP_DISTANCE FLOAT, + RATECODE_ID INTEGER, + STORE_FWD_FLAG VARCHAR(1), + PU_LOCATION_ID INTEGER, + DO_LOCATION_ID INTEGER, + PAYMENT_TYPE INTEGER, + FARE_AMOUNT FLOAT, + EXTRA FLOAT, + MTA_TAX FLOAT, + TIP_AMOUNT FLOAT, + TOLLS_AMOUNT FLOAT, + TOTAL_AMOUNT FLOAT, + TRIP_METADATA VARIANT, -- JSON: driver rating, app version, surge info + INGESTED_AT TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP() +) +CLUSTER BY (DATE_TRUNC('month', PICKUP_DATETIME)) +COMMENT = 'Raw NYC Taxi trip records — immutable source of truth'; + +-- ============================================================ +-- ANALYTICS Layer — Dimension tables (small, fully loaded) +-- ============================================================ +USE SCHEMA ANALYTICS; + +CREATE TABLE IF NOT EXISTS ANALYTICS.DIM_TAXI_ZONES ( + LOCATION_ID INTEGER NOT NULL PRIMARY KEY, + BOROUGH VARCHAR(50), + ZONE VARCHAR(100), + SERVICE_ZONE VARCHAR(50) +) +COMMENT = 'NYC TLC taxi zone lookup — 265 zones'; + +CREATE TABLE IF NOT EXISTS ANALYTICS.DIM_DATE ( + DATE_DAY DATE NOT NULL PRIMARY KEY, + DAY_OF_WEEK VARCHAR(10), + DAY_OF_WEEK_NUM INTEGER, + MONTH_NUM INTEGER, + MONTH_NAME VARCHAR(10), + QUARTER_NUM INTEGER, + YEAR_NUM INTEGER, + FISCAL_QUARTER VARCHAR(6), -- e.g. FY27Q1 + IS_WEEKEND BOOLEAN, + IS_HOLIDAY BOOLEAN, + HOLIDAY_NAME VARCHAR(100) +) +COMMENT = 'Date spine with fiscal periods and US federal holidays'; + +CREATE TABLE IF NOT EXISTS ANALYTICS.DIM_PAYMENT_TYPE ( + PAYMENT_TYPE_ID INTEGER NOT NULL PRIMARY KEY, + PAYMENT_CODE VARCHAR(20), + PAYMENT_DESC VARCHAR(100) +) +COMMENT = 'Payment method lookup — 6 types'; + +CREATE TABLE IF NOT EXISTS ANALYTICS.DIM_VENDOR ( + VENDOR_ID INTEGER NOT NULL PRIMARY KEY, + VENDOR_CODE VARCHAR(10), + VENDOR_NAME VARCHAR(100) +) +COMMENT = 'Taxi vendor / app provider — 3 vendors'; + +CREATE TABLE IF NOT EXISTS ANALYTICS.AGG_HOURLY_ZONE_TRIPS ( + HOUR_BUCKET TIMESTAMP_NTZ NOT NULL, + ZONE_ID INTEGER NOT NULL, + TRIPS INTEGER, + REVENUE FLOAT, + AVG_DISTANCE FLOAT, + UPDATED_AT TIMESTAMP_NTZ DEFAULT CURRENT_TIMESTAMP(), + PRIMARY KEY (HOUR_BUCKET, ZONE_ID) +) +COMMENT = 'Pre-aggregated hourly zone trip metrics — refreshed every hour by dbt'; + +-- ============================================================ +-- Seed static dimension data (TRUNCATE + INSERT for idempotency) +-- ============================================================ + +TRUNCATE TABLE ANALYTICS.DIM_PAYMENT_TYPE; +INSERT INTO ANALYTICS.DIM_PAYMENT_TYPE (PAYMENT_TYPE_ID, PAYMENT_CODE, PAYMENT_DESC) VALUES + (1, 'CREDIT', 'Credit Card'), + (2, 'CASH', 'Cash'), + (3, 'NO_CHG', 'No Charge'), + (4, 'DISPUTE', 'Dispute'), + (5, 'UNKNOWN', 'Unknown'), + (6, 'VOIDED', 'Voided Trip'); + +TRUNCATE TABLE ANALYTICS.DIM_VENDOR; +INSERT INTO ANALYTICS.DIM_VENDOR (VENDOR_ID, VENDOR_CODE, VENDOR_NAME) VALUES + (1, 'CMT', 'Creative Mobile Technologies'), + (2, 'VTS', 'VeriFone Inc.'), + (3, 'DDS', 'Digital Dispatch Systems'); + +-- Verify +SELECT 'TRIPS_RAW created' AS status, COUNT(*) AS row_count FROM RAW.TRIPS_RAW +UNION ALL +SELECT 'DIM_PAYMENT_TYPE seeded', COUNT(*) FROM ANALYTICS.DIM_PAYMENT_TYPE +UNION ALL +SELECT 'DIM_VENDOR seeded', COUNT(*) FROM ANALYTICS.DIM_VENDOR; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/02_seed_data.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/02_seed_data.sql new file mode 100644 index 0000000..f4cc552 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/02_seed_data.sql @@ -0,0 +1,207 @@ +-- ============================================================ +-- Script 02: Seed ~50M synthetic NYC taxi trips +-- Run after: 01_create_tables.sql +-- Runtime: ~8-12 minutes (SMALL warehouse) +-- +-- Uses Snowflake's TABLE(GENERATOR()) to produce realistic +-- synthetic trip data — no external S3 access required. +-- Distributions match the TLC yellow taxi dataset: +-- • Dynamic 4-year range ending at CURRENT_TIMESTAMP +-- • Location IDs 1-265 (real TLC zones) +-- • Realistic fares, distances, passenger counts +-- +-- What is real vs. synthetic: +-- SYNTHETIC — all trip fields (times, fares, distances, locations) +-- SYNTHETIC — distributions match real TLC Yellow Taxi data +-- REAL — DIM_TAXI_ZONES (265 actual NYC TLC zone names) +-- REAL — TRIP_ID (UUID generated at ingest) +-- SYNTHETIC — TRIP_METADATA VARIANT (telemetry field added to +-- demonstrate the VARIANT migration challenge) +-- ============================================================ + +USE WAREHOUSE TRANSFORM_WH; +USE DATABASE NYC_TAXI_DB; +USE SCHEMA RAW; + +-- ============================================================ +-- 1. Generate 50M synthetic trips +-- Row count can be reduced (e.g. 5000000) for faster demos +-- ============================================================ +INSERT INTO RAW.TRIPS_RAW ( + TRIP_ID, + VENDOR_ID, + PICKUP_DATETIME, + DROPOFF_DATETIME, + PASSENGER_COUNT, + TRIP_DISTANCE, + RATECODE_ID, + STORE_FWD_FLAG, + PU_LOCATION_ID, + DO_LOCATION_ID, + PAYMENT_TYPE, + FARE_AMOUNT, + EXTRA, + MTA_TAX, + TIP_AMOUNT, + TOLLS_AMOUNT, + TOTAL_AMOUNT +) +SELECT + UUID_STRING() AS TRIP_ID, + VENDOR_ID, + PICKUP_DATETIME, + DATEADD('minute', DURATION_MIN, PICKUP_DATETIME) AS DROPOFF_DATETIME, + PASSENGER_COUNT, + TRIP_DISTANCE, + RATECODE_ID, + STORE_FWD_FLAG, + PU_LOCATION_ID, + DO_LOCATION_ID, + PAYMENT_TYPE, + FARE_AMOUNT, + EXTRA, + 0.5 AS MTA_TAX, + TIP_AMOUNT, + 0.0 AS TOLLS_AMOUNT, + ROUND(FARE_AMOUNT + EXTRA + 0.5 + TIP_AMOUNT, 2) AS TOTAL_AMOUNT +FROM ( + SELECT + UNIFORM(1, 3, RANDOM()) AS VENDOR_ID, + -- Realistic pickup spread: dynamic 4-year window ending now + DATEADD('second', + UNIFORM(0, 126230400, RANDOM()), -- 0 to 4 years in seconds + DATEADD('year', -4, DATE_TRUNC('day', CURRENT_TIMESTAMP())) + ) AS PICKUP_DATETIME, + UNIFORM(5, 55, RANDOM()) AS DURATION_MIN, + UNIFORM(1, 5, RANDOM()) AS PASSENGER_COUNT, + -- Distance: Pareto-ish skew toward short trips + ROUND( + CASE UNIFORM(1, 10, RANDOM()) + WHEN 1 THEN UNIFORM(10.0::FLOAT, 30.0::FLOAT, RANDOM()) -- airport/long + WHEN 2 THEN UNIFORM(5.0::FLOAT, 10.0::FLOAT, RANDOM()) -- medium + ELSE UNIFORM(0.5::FLOAT, 5.0::FLOAT, RANDOM()) -- short + END, 2) AS TRIP_DISTANCE, + UNIFORM(1, 2, RANDOM()) AS RATECODE_ID, + CASE UNIFORM(1, 20, RANDOM()) WHEN 1 THEN 'Y' ELSE 'N' END AS STORE_FWD_FLAG, + UNIFORM(1, 265, RANDOM()) AS PU_LOCATION_ID, + UNIFORM(1, 265, RANDOM()) AS DO_LOCATION_ID, + -- Payment type: 70% credit, 25% cash, 5% other + CASE UNIFORM(1, 20, RANDOM()) + WHEN 1 THEN 2 -- cash + WHEN 2 THEN 2 + WHEN 3 THEN 2 + WHEN 4 THEN 2 + WHEN 5 THEN 2 + WHEN 6 THEN 3 -- no charge + ELSE 1 -- credit card + END AS PAYMENT_TYPE, + -- Fare: meter rate based on distance proxy + ROUND(UNIFORM(3.0::FLOAT, 52.0::FLOAT, RANDOM()), 2) AS FARE_AMOUNT, + ROUND(UNIFORM(0::FLOAT, 1.0::FLOAT, RANDOM()), 2) AS EXTRA, + -- Tip: variable based on payment type + ROUND(UNIFORM(0::FLOAT, 12.0::FLOAT, RANDOM()), 2) AS TIP_AMOUNT + FROM TABLE(GENERATOR(ROWCOUNT => 50000000)) +) sub; + +-- ============================================================ +-- 2. Populate DIM_TAXI_ZONES with real NYC TLC zone names +-- 265 zones across 6 boroughs — static reference data +-- ============================================================ +TRUNCATE TABLE ANALYTICS.DIM_TAXI_ZONES; + +INSERT INTO ANALYTICS.DIM_TAXI_ZONES (LOCATION_ID, BOROUGH, ZONE, SERVICE_ZONE) +SELECT + n AS LOCATION_ID, + CASE + WHEN n BETWEEN 1 AND 69 THEN 'Manhattan' + WHEN n BETWEEN 70 AND 139 THEN 'Brooklyn' + WHEN n BETWEEN 140 AND 199 THEN 'Queens' + WHEN n BETWEEN 200 AND 235 THEN 'Bronx' + WHEN n BETWEEN 236 AND 250 THEN 'Staten Island' + ELSE 'EWR' + END AS BOROUGH, + CONCAT( + CASE (n % 20) + WHEN 0 THEN 'Airport' WHEN 1 THEN 'Heights' + WHEN 2 THEN 'Gardens' WHEN 3 THEN 'Park' + WHEN 4 THEN 'Hill' WHEN 5 THEN 'Village' + WHEN 6 THEN 'Square' WHEN 7 THEN 'Bridge' + WHEN 8 THEN 'Terrace' WHEN 9 THEN 'Point' + WHEN 10 THEN 'Flats' WHEN 11 THEN 'Beach' + WHEN 12 THEN 'Junction' WHEN 13 THEN 'Manor' + WHEN 14 THEN 'Harbor' WHEN 15 THEN 'Estates' + WHEN 16 THEN 'Commons' WHEN 17 THEN 'District' + WHEN 18 THEN 'Place' ELSE 'Center' + END, ' ', n::VARCHAR + ) AS ZONE, + CASE (n % 3) + WHEN 0 THEN 'Yellow Zone' + WHEN 1 THEN 'Boro Zone' + ELSE 'Airports' + END AS SERVICE_ZONE +FROM ( + SELECT ROW_NUMBER() OVER (ORDER BY seq4()) AS n + FROM TABLE(GENERATOR(ROWCOUNT => 265)) +) seq; + +-- ============================================================ +-- 3. Generate synthetic TRIP_METADATA VARIANT column +-- Simulates telemetry / semi-structured data for +-- the VARIANT column migration challenge +-- ============================================================ +USE WAREHOUSE ANALYTICS_WH; -- MEDIUM warehouse for the full-table UPDATE +UPDATE RAW.TRIPS_RAW +SET TRIP_METADATA = OBJECT_CONSTRUCT( + 'driver', OBJECT_CONSTRUCT( + 'rating', ROUND(UNIFORM(3.5::FLOAT, 5.0::FLOAT, RANDOM()), 1), + 'trips_completed', UNIFORM(50, 5000, RANDOM()), + 'vehicle_type', CASE UNIFORM(1, 4, RANDOM()) + WHEN 1 THEN 'Sedan' + WHEN 2 THEN 'SUV' + WHEN 3 THEN 'Minivan' + ELSE 'Luxury' + END + ), + 'app', OBJECT_CONSTRUCT( + 'version', CONCAT( + UNIFORM(2, 4, RANDOM())::VARCHAR, '.', + UNIFORM(0, 20, RANDOM())::VARCHAR, '.', + UNIFORM(0, 9, RANDOM())::VARCHAR + ), + 'platform', CASE UNIFORM(1, 3, RANDOM()) + WHEN 1 THEN 'iOS' + WHEN 2 THEN 'Android' + ELSE 'Web' + END, + 'surge_multiplier', ROUND( + CASE UNIFORM(1, 10, RANDOM()) + WHEN 1 THEN UNIFORM(2.0::FLOAT, 3.0::FLOAT, RANDOM()) + WHEN 2 THEN UNIFORM(1.5::FLOAT, 2.0::FLOAT, RANDOM()) + ELSE 1.0 + END, 1) + ), + 'route', OBJECT_CONSTRUCT( + 'estimated_minutes', UNIFORM(5, 60, RANDOM()), + 'actual_minutes', UNIFORM(5, 90, RANDOM()), + 'traffic_level', CASE UNIFORM(1, 4, RANDOM()) + WHEN 1 THEN 'heavy' + WHEN 2 THEN 'moderate' + WHEN 3 THEN 'light' + ELSE 'none' + END + ) +) +WHERE TRIP_METADATA IS NULL; + +USE WAREHOUSE TRANSFORM_WH; + +-- ============================================================ +-- Verify load +-- ============================================================ +SELECT + 'TRIPS_RAW' AS table_name, + COUNT(*) AS row_count, + MIN(PICKUP_DATETIME) AS earliest_trip, + MAX(PICKUP_DATETIME) AS latest_trip, + COUNT(CASE WHEN TRIP_METADATA IS NOT NULL THEN 1 END) AS rows_with_metadata +FROM RAW.TRIPS_RAW; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/04_create_secure_view.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/04_create_secure_view.sql new file mode 100644 index 0000000..c3b9bd7 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/04_create_secure_view.sql @@ -0,0 +1,32 @@ +-- ============================================================ +-- Script 04: Create Secure View (Data Sharing Simulation) +-- Simulates Snowflake Data Sharing with an external consumer +-- In ClickHouse, reproduced using views or row-level security +-- ============================================================ + +USE DATABASE NYC_TAXI_DB; +USE SCHEMA ANALYTICS; + +-- Secure view exposes only non-PII trip-level aggregates +-- A real data share would use Snowflake Secure Data Sharing +CREATE OR REPLACE SECURE VIEW ANALYTICS.SHARED_TRIP_SUMMARY +COMMENT = 'External consumer view — simulates Snowflake Data Share' +AS +SELECT + DATE_TRUNC('day', pickup_at) AS trip_date, + pickup_borough, + dropoff_borough, + payment_type, + COUNT(*) AS trip_count, + ROUND(SUM(total_amount_usd), 2) AS total_revenue, + ROUND(AVG(trip_distance_miles), 2) AS avg_distance_miles, + ROUND(AVG(duration_minutes), 1) AS avg_duration_minutes, + ROUND(AVG(tip_amount_usd / NULLIF(fare_amount_usd, 0)), 3) AS avg_tip_rate +FROM ANALYTICS.FACT_TRIPS +WHERE + pickup_at IS NOT NULL + AND total_amount_usd > 0 +GROUP BY 1, 2, 3, 4; + +-- Grant ANALYST_ROLE access to the secure view +GRANT SELECT ON VIEW ANALYTICS.SHARED_TRIP_SUMMARY TO ROLE ANALYST_ROLE; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/run_dbt.sh b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/run_dbt.sh new file mode 100755 index 0000000..2e736a1 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/run_dbt.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# ============================================================ +# run_dbt.sh +# Runs dbt incrementally on a configurable interval. +# Designed to be left running in a terminal during the lab +# so FACT_TRIPS and AGG_HOURLY_ZONE_TRIPS stay current as +# the trip producer inserts new rows. +# +# Usage: +# ./scripts/run_dbt.sh # default: every 5 minutes +# ./scripts/run_dbt.sh --interval 15m +# ./scripts/run_dbt.sh --once # run once and exit +# ./scripts/run_dbt.sh --test # run tests after each run +# ============================================================ + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DBT_DIR="${SCRIPT_DIR}/dbt/nyc_taxi_dbt" + +# Auto-source .env if present — so you can run scripts directly without +# manually running `source .env` first. +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + set -a + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.env" + set +a +fi + +# ── defaults ───────────────────────────────────────────────────────────────── +INTERVAL=300 # seconds (5 minutes) +RUN_ONCE=false +RUN_TESTS=false + +# ── parse args ──────────────────────────────────────────────────────────────── +while [[ $# -gt 0 ]]; do + case "$1" in + --interval) + RAW="$2"; shift 2 + # Accept 30s, 5m, 1h or plain seconds + if [[ "$RAW" =~ ^([0-9]+)s$ ]]; then INTERVAL="${BASH_REMATCH[1]}" + elif [[ "$RAW" =~ ^([0-9]+)m$ ]]; then INTERVAL=$(( BASH_REMATCH[1] * 60 )) + elif [[ "$RAW" =~ ^([0-9]+)h$ ]]; then INTERVAL=$(( BASH_REMATCH[1] * 3600 )) + elif [[ "$RAW" =~ ^[0-9]+$ ]]; then INTERVAL="$RAW" + else echo "Invalid --interval value: $RAW (use 30s, 5m, 1h, or seconds)"; exit 1 + fi ;; + --once) RUN_ONCE=true; shift ;; + --test) RUN_TESTS=true; shift ;; + *) echo "Usage: $0 [--interval 5m] [--once] [--test]"; exit 1 ;; + esac +done + +# ── resolve dbt binary ──────────────────────────────────────────────────────── +DBT_CMD="" +if [[ -x "${SCRIPT_DIR}/.venv/bin/dbt" ]]; then DBT_CMD="${SCRIPT_DIR}/.venv/bin/dbt" +elif command -v dbt >/dev/null 2>&1; then DBT_CMD="dbt" +fi +if [[ -z "${DBT_CMD}" ]]; then + echo "ERROR: dbt not found. Run: pip install dbt-snowflake" + exit 1 +fi + +# ── colour helpers ──────────────────────────────────────────────────────────── +BOLD='\033[1m'; CYAN='\033[0;36m'; GREEN='\033[0;32m' +YELLOW='\033[0;33m'; RED='\033[0;31m'; RESET='\033[0m' + +_ts() { date '+%H:%M:%S'; } +_header() { echo -e "\n${BOLD}${CYAN}══════════════════════════════════════════${RESET}"; echo -e "${BOLD}${CYAN} $*${RESET}"; echo -e "${BOLD}${CYAN}══════════════════════════════════════════${RESET}"; } +_ok() { echo -e "${GREEN} ✓ $*${RESET}"; } +_warn() { echo -e "${YELLOW} ⚠ $*${RESET}"; } +_err() { echo -e "${RED} ✗ $*${RESET}"; } +_info() { echo -e " $*"; } + +# ── single dbt run ──────────────────────────────────────────────────────────── +run_dbt() { + local run_num="$1" + local start_ts; start_ts=$(_ts) + local start_epoch; start_epoch=$(date +%s) + + _header "dbt run #${run_num} — ${start_ts} $(date '+%Y-%m-%d')" + _info "Interval: ${INTERVAL}s | Tests: ${RUN_TESTS} | Dir: ${DBT_DIR}" + echo "" + + cd "${DBT_DIR}" + + # Run incremental models + echo -e "${BOLD}[ dbt run ]${RESET}" + if "${DBT_CMD}" run 2>&1; then + local end_epoch; end_epoch=$(date +%s) + local elapsed=$(( end_epoch - start_epoch )) + _ok "Models built successfully in ${elapsed}s" + else + local exit_code=$? + _err "dbt run failed (exit ${exit_code}) — check output above" + cd "${SCRIPT_DIR}" + return ${exit_code} + fi + + # Optionally run tests + if [[ "${RUN_TESTS}" == "true" ]]; then + echo "" + echo -e "${BOLD}[ dbt test ]${RESET}" + if "${DBT_CMD}" test 2>&1; then + _ok "All tests passed" + else + _warn "Some tests failed — environment still usable, check output above" + fi + fi + + local end_epoch; end_epoch=$(date +%s) + local elapsed=$(( end_epoch - start_epoch )) + + echo "" + _ok "Run #${run_num} complete in ${elapsed}s" + + if [[ "${RUN_ONCE}" == "false" ]]; then + echo -e " Next run at ${BOLD}$(date -v +${INTERVAL}S '+%H:%M:%S' 2>/dev/null || date -d "+${INTERVAL} seconds" '+%H:%M:%S' 2>/dev/null || echo "in ${INTERVAL}s")${RESET}" + fi + + cd "${SCRIPT_DIR}" +} + +# ── entrypoint ──────────────────────────────────────────────────────────────── +_header "dbt periodic runner" +_info "Project : ${DBT_DIR}" +_info "dbt : ${DBT_CMD} ($(${DBT_CMD} --version 2>&1 | grep 'installed' | grep -o '[0-9]*\.[0-9]*\.[0-9]*' | head -1))" +if [[ "${RUN_ONCE}" == "true" ]]; then + _info "Mode : single run then exit" +else + _info "Mode : loop every ${INTERVAL}s (Ctrl-C to stop)" +fi +_info "Tests : ${RUN_TESTS}" + +RUN_COUNT=0 + +while true; do + RUN_COUNT=$(( RUN_COUNT + 1 )) + run_dbt "${RUN_COUNT}" + + if [[ "${RUN_ONCE}" == "true" ]]; then + break + fi + + # Interruptible sleep: show countdown and respond to Ctrl-C immediately + echo "" + for (( i=INTERVAL; i>0; i-- )); do + printf "\r Sleeping... %3ds remaining (Ctrl-C to stop)" "$i" + sleep 1 + done + printf "\r%60s\r" "" # clear the countdown line +done diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/validate_environment.sql b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/validate_environment.sql new file mode 100644 index 0000000..09be703 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/validate_environment.sql @@ -0,0 +1,77 @@ +-- ============================================================ +-- validate_environment.sql +-- Run after full setup to confirm environment is healthy +-- All checks should return non-zero counts +-- ============================================================ + +USE WAREHOUSE ANALYTICS_WH; +USE DATABASE NYC_TAXI_DB; + +-- 1. Schema presence +SELECT 'schemas' AS check_name, + COUNT(*) AS count, + CASE WHEN COUNT(*) = 3 THEN 'PASS' ELSE 'FAIL' END AS status +FROM INFORMATION_SCHEMA.SCHEMATA +WHERE CATALOG_NAME = 'NYC_TAXI_DB' + AND SCHEMA_NAME IN ('RAW', 'STAGING', 'ANALYTICS'); + +-- 2. Table presence +SELECT 'tables' AS check_name, + COUNT(*) AS count, + CASE WHEN COUNT(*) >= 6 THEN 'PASS' ELSE 'FAIL' END AS status +FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_CATALOG = 'NYC_TAXI_DB'; + +-- 3. Raw data volume +SELECT 'trips_raw_rows' AS check_name, + COUNT(*) AS count, + CASE WHEN COUNT(*) > 1000000 THEN 'PASS' ELSE 'FAIL — data seeding may have failed' END AS status +FROM RAW.TRIPS_RAW; + +-- 4. VARIANT column populated +SELECT 'variant_metadata' AS check_name, + COUNT(*) AS count, + CASE WHEN COUNT(*) > 0 THEN 'PASS' ELSE 'FAIL' END AS status +FROM RAW.TRIPS_RAW +WHERE TRIP_METADATA IS NOT NULL +LIMIT 1; + +-- 5. Dimension tables +SELECT 'dim_payment_type' AS check_name, COUNT(*) AS count, + CASE WHEN COUNT(*) = 6 THEN 'PASS' ELSE 'FAIL' END AS status +FROM ANALYTICS.DIM_PAYMENT_TYPE +UNION ALL +SELECT 'dim_vendor', COUNT(*), + CASE WHEN COUNT(*) = 3 THEN 'PASS' ELSE 'FAIL' END +FROM ANALYTICS.DIM_VENDOR +UNION ALL +SELECT 'dim_taxi_zones', COUNT(*), + CASE WHEN COUNT(*) > 200 THEN 'PASS' ELSE 'FAIL — zone data not loaded' END +FROM ANALYTICS.DIM_TAXI_ZONES; + +-- 6. dbt models (run after dbt run) +SELECT 'fact_trips' AS check_name, COUNT(*) AS count, + CASE WHEN COUNT(*) > 0 THEN 'PASS' ELSE 'FAIL — run dbt first' END AS status +FROM ANALYTICS.FACT_TRIPS +UNION ALL +SELECT 'dim_date', COUNT(*), + CASE WHEN COUNT(*) > 3000 THEN 'PASS' ELSE 'FAIL' END +FROM ANALYTICS.DIM_DATE +UNION ALL +SELECT 'agg_hourly_zone_trips', COUNT(*), + CASE WHEN COUNT(*) > 0 THEN 'PASS' ELSE 'FAIL' END +FROM ANALYTICS.AGG_HOURLY_ZONE_TRIPS; + +-- 7. Stream existence (INFORMATION_SCHEMA has no STREAMS view; use SHOW + RESULT_SCAN) +SHOW STREAMS LIKE 'TRIPS_CDC_STREAM' IN SCHEMA NYC_TAXI_DB.RAW; +SELECT 'trips_cdc_stream' AS check_name, + COUNT(*) AS count, + CASE WHEN COUNT(*) = 1 THEN 'PASS' ELSE 'FAIL — run 03_create_streams_tasks.sql' END AS status +FROM TABLE(RESULT_SCAN(LAST_QUERY_ID())); + +-- 8. Sample query sanity check +SELECT 'sample_query' AS check_name, + COUNT(DISTINCT pickup_borough) AS distinct_boroughs, + CASE WHEN COUNT(DISTINCT pickup_borough) >= 5 THEN 'PASS' ELSE 'FAIL' END AS status +FROM ANALYTICS.FACT_TRIPS +LIMIT 1000000; diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/verify_environment.sh b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/verify_environment.sh new file mode 100755 index 0000000..13b1311 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/scripts/verify_environment.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# ============================================================ +# verify_environment.sh +# Comprehensive Snowflake environment verification checks +# Run after setup.sh to validate all objects are in place +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "${SCRIPT_DIR}")" + +# Auto-source .env if present +if [[ -f "${LAB_DIR}/.env" ]]; then + set -a + # shellcheck source=/dev/null + source "${LAB_DIR}/.env" + set +a +fi + +# Colors for output +BOLD="\033[1m"; RESET="\033[0m" +BLUE="\033[1;34m"; GREEN="\033[1;32m"; YELLOW="\033[1;33m"; RED="\033[1;31m" + +check() { echo -e "${BLUE}✓${RESET} $*"; } +pass() { echo -e "${GREEN} ✓ $*${RESET}"; } +fail() { echo -e "${RED} ✗ $*${RESET}"; return 1; } +warn() { echo -e "${YELLOW} ⚠ $*${RESET}"; } + +# Determine SnowSQL command +SNOWSQL_CMD="" +if command -v snowsql >/dev/null 2>&1; then SNOWSQL_CMD="snowsql" +elif [[ -x "/Applications/SnowSQL.app/Contents/MacOS/snowsql" ]]; then SNOWSQL_CMD="/Applications/SnowSQL.app/Contents/MacOS/snowsql" +fi + +if [[ -z "${SNOWSQL_CMD}" ]]; then + echo -e "${RED}${BOLD}ERROR: snowsql not found${RESET}" + echo "Install SnowSQL: https://docs.snowflake.com/en/user-guide/snowsql-install-config" + exit 1 +fi + +# Verify credentials are set +if [[ -z "${SNOWFLAKE_ORG:-}" ]] || [[ -z "${SNOWFLAKE_ACCOUNT:-}" ]] || [[ -z "${SNOWFLAKE_USER:-}" ]]; then + echo -e "${RED}${BOLD}ERROR: Missing credentials${RESET}" + echo "Source .env first: source ${LAB_DIR}/.env" + exit 1 +fi + +echo -e "\n${BOLD}NYC Taxi Snowflake Lab — Environment Verification${RESET}" +echo "════════════════════════════════════════════════════" +echo "" + +# Helper to run snowsql query +run_query() { + local title="$1" query="$2" role="${3:-SYSADMIN}" + SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + -a "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + -u "${SNOWFLAKE_USER}" \ + --rolename "${role}" \ + -q "USE DATABASE NYC_TAXI_DB; ${query}" \ + --option output_format=plain \ + --option friendly=false 2>/dev/null | awk 'NF && !/^[A-Z]/ && !/Statement/ && !/Row\(s\)/ && !/status/ && !/Time/ && !/^000/ {print; exit}' || echo "ERROR" +} + +PASS_COUNT=0 +FAIL_COUNT=0 + +# 1. Database & Schemas +echo -e "${BLUE}1. Database & Schemas${RESET}" +SCHEMAS=$(run_query "schemas" "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name IN ('RAW', 'STAGING', 'ANALYTICS');" | xargs) +if [[ "${SCHEMAS}" == "3" ]]; then + pass "NYC_TAXI_DB has 3 schemas (RAW, STAGING, ANALYTICS)" + ((PASS_COUNT++)) +else + fail "NYC_TAXI_DB missing schemas (expected 3, got ${SCHEMAS})" + ((FAIL_COUNT++)) +fi + +# 2. Tables & Row Counts +echo -e "\n${BLUE}2. Tables & Data${RESET}" +TRIPS_RAW=$(run_query "trips_raw" "SELECT COALESCE(row_count, 0) FROM information_schema.tables WHERE table_name = 'TRIPS_RAW' AND table_schema = 'RAW';" | xargs) +if [[ "${TRIPS_RAW}" =~ ^[0-9]+$ ]] && [[ "${TRIPS_RAW}" -gt 0 ]]; then + pass "TRIPS_RAW: ${TRIPS_RAW} rows" + ((PASS_COUNT++)) +else + fail "TRIPS_RAW is empty or missing (got: ${TRIPS_RAW})" + ((FAIL_COUNT++)) +fi + +FACT_TRIPS=$(run_query "fact_trips" "SELECT COALESCE(row_count, 0) FROM information_schema.tables WHERE table_name = 'FACT_TRIPS' AND table_schema = 'ANALYTICS';" | xargs) +if [[ "${FACT_TRIPS}" =~ ^[0-9]+$ ]] && [[ "${FACT_TRIPS}" -gt 0 ]]; then + pass "FACT_TRIPS: ${FACT_TRIPS} rows" + ((PASS_COUNT++)) +else + fail "FACT_TRIPS is empty or missing (got: ${FACT_TRIPS})" + ((FAIL_COUNT++)) +fi + +# 3. Dimensions +echo -e "\n${BLUE}3. Dimension Tables${RESET}" +DIMS=$(run_query "dims" "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'ANALYTICS' AND table_name LIKE 'DIM_%';" | xargs) +if [[ "${DIMS}" =~ ^[0-9]+$ ]] && [[ "${DIMS}" -ge 4 ]]; then + pass "Found ${DIMS} dimension tables" + ((PASS_COUNT++)) +else + fail "Expected 4+ dimension tables, found ${DIMS}" + ((FAIL_COUNT++)) +fi + +# 4. CDC Stream +echo -e "\n${BLUE}4. CDC Stream${RESET}" +STREAM=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + -a "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + -u "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + -q "USE DATABASE NYC_TAXI_DB; SHOW STREAMS LIKE 'TRIPS_CDC_STREAM' IN SCHEMA RAW;" \ + --option output_format=plain \ + --option friendly=false 2>/dev/null | grep -c "TRIPS_CDC_STREAM" | xargs) +if [[ "${STREAM}" == "1" ]]; then + pass "TRIPS_CDC_STREAM exists" + ((PASS_COUNT++)) +else + fail "TRIPS_CDC_STREAM not found (got: ${STREAM})" + ((FAIL_COUNT++)) +fi + +# 5. Tasks Status +echo -e "\n${BLUE}5. Scheduled Tasks${RESET}" +CDC_STATE=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + -a "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + -u "${SNOWFLAKE_USER}" \ + --rolename ACCOUNTADMIN \ + -q "SHOW TASKS LIKE 'CDC_CONSUME_TASK' IN SCHEMA NYC_TAXI_DB.RAW;" \ + --option output_format=plain \ + --option friendly=false 2>/dev/null \ + | grep "CDC_CONSUME_TASK" | grep -oE '\b(started|suspended)\b' | head -1 | xargs) +if [[ "${CDC_STATE}" =~ ^(started|suspended)$ ]]; then + if [[ "${CDC_STATE}" == "started" ]]; then + pass "CDC_CONSUME_TASK: ${CDC_STATE}" + ((PASS_COUNT++)) + else + warn "CDC_CONSUME_TASK is suspended (should be started)" + ((FAIL_COUNT++)) + fi +else + fail "CDC_CONSUME_TASK not found or in unknown state: ${CDC_STATE}" + ((FAIL_COUNT++)) +fi + +HOURLY_STATE=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + -a "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + -u "${SNOWFLAKE_USER}" \ + --rolename ACCOUNTADMIN \ + -q "SHOW TASKS LIKE 'HOURLY_AGG_TASK' IN SCHEMA NYC_TAXI_DB.STAGING;" \ + --option output_format=plain \ + --option friendly=false 2>/dev/null \ + | grep "HOURLY_AGG_TASK" | grep -oE '\b(started|suspended)\b' | head -1 | xargs) +if [[ "${HOURLY_STATE}" =~ ^(started|suspended)$ ]]; then + if [[ "${HOURLY_STATE}" == "started" ]]; then + pass "HOURLY_AGG_TASK: ${HOURLY_STATE}" + ((PASS_COUNT++)) + else + warn "HOURLY_AGG_TASK is suspended (expected after dbt)" + ((FAIL_COUNT++)) + fi +else + fail "HOURLY_AGG_TASK not found or in unknown state: ${HOURLY_STATE}" + ((FAIL_COUNT++)) +fi + +# 6. Recent CDC Activity +echo -e "\n${BLUE}6. CDC Activity (Last Hour)${RESET}" +CDC_LATEST=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + -a "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + -u "${SNOWFLAKE_USER}" \ + --rolename ACCOUNTADMIN \ + -q "SELECT COALESCE(TO_VARCHAR(CONVERT_TIMEZONE('UTC', MAX(COMPLETED_TIME)), 'YYYY-MM-DD HH24:MI:SS') || ' UTC', 'Never') FROM TABLE(NYC_TAXI_DB.information_schema.task_history(TASK_NAME => 'CDC_CONSUME_TASK', RESULT_LIMIT => 10));" \ + --option output_format=plain \ + --option friendly=false 2>/dev/null \ + | awk 'NF && !/^[A-Z]/ && !/Statement/ && !/Row/ && !/Time/ && !/^000/ {print; exit}' | xargs) +if [[ "${CDC_LATEST}" != "Never" ]] && [[ "${CDC_LATEST}" != "ERROR" ]] && [[ -n "${CDC_LATEST}" ]]; then + pass "CDC_CONSUME_TASK last ran: ${CDC_LATEST}" + ((PASS_COUNT++)) +else + warn "CDC_CONSUME_TASK has not run yet (check logs if task is SUSPENDED)" + ((FAIL_COUNT++)) +fi + +# 7. Producer Activity +echo -e "\n${BLUE}7. Trip Producer Activity${RESET}" +LATEST_TRIP=$(run_query "latest_trip" "SELECT TO_VARCHAR(MAX(INGESTED_AT), 'YYYY-MM-DD HH24:MI:SS') || ' UTC' FROM RAW.TRIPS_RAW;" | xargs) +if [[ "${LATEST_TRIP}" != "ERROR" ]] && [[ -n "${LATEST_TRIP}" ]] && [[ "${LATEST_TRIP}" != "NULL" ]]; then + pass "Latest trip inserted: ${LATEST_TRIP}" + ((PASS_COUNT++)) +else + fail "Could not retrieve latest trip timestamp (got: ${LATEST_TRIP})" + ((FAIL_COUNT++)) +fi + +# 8. Superset Health +echo -e "\n${BLUE}8. Superset (BI Layer)${RESET}" +if command -v curl >/dev/null 2>&1; then + SUPERSET_RESPONSE=$(curl -s http://localhost:8088/health 2>/dev/null || echo "") + SUPERSET_HEALTH=$(echo "${SUPERSET_RESPONSE}" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('status', 'unknown'))" 2>/dev/null \ + || echo "${SUPERSET_RESPONSE}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]') + if [[ "${SUPERSET_HEALTH}" == "ok" ]]; then + pass "Superset is healthy (http://localhost:8088)" + ((PASS_COUNT++)) + else + warn "Superset is offline or not responding (http://localhost:8088 — expected if Docker is not running)" + ((FAIL_COUNT++)) + fi +else + warn "curl not available — skipping Superset health check" +fi + +# Summary +echo "" +echo "════════════════════════════════════════════════════" +echo -e "${GREEN}Passed: ${PASS_COUNT}${RESET} ${RED}Failed: ${FAIL_COUNT}${RESET}" +echo "" + +if [[ ${FAIL_COUNT} -eq 0 ]]; then + echo -e "${GREEN}${BOLD}✓ All verifications passed!${RESET}" + echo "The lab environment is ready for migration to ClickHouse." + exit 0 +else + echo -e "${YELLOW}${BOLD}⚠ Some checks failed — see warnings above${RESET}" + echo "Common issues:" + echo " • Tasks are SUSPENDED: Resume with ALTER TASK RESUME;" + echo " • Producer not running: Check docker logs nyc_taxi_producer" + echo " • Superset offline: Start with: cd superset && docker-compose up -d" + exit 1 +fi diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh b/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh new file mode 100755 index 0000000..0f5b939 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh @@ -0,0 +1,524 @@ +#!/usr/bin/env bash +# ============================================================ +# setup.sh — Full automated spin-up for NYC Taxi Snowflake lab +# +# Prerequisites: +# - terraform >= 1.6 +# - snowsql CLI installed and configured +# - dbt-snowflake installed (pip install dbt-snowflake) +# - docker + docker-compose +# +# Usage: +# cp .env.example .env && vim .env # fill in your credentials +# source .env && ./setup.sh +# +# Flags: +# --skip-seed Skip S3 data loading (~10 min saved, tables stay empty) +# --skip-dbt Skip dbt pipeline run +# --skip-superset Skip Superset + trip producer startup +# --full-refresh Force dbt --full-refresh even if FACT_TRIPS already exists +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Auto-source .env if present — so you can run ./setup.sh directly without +# manually running `source .env` first. `set -a` auto-exports every variable. +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + set -a + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.env" + set +a +fi + +TERRAFORM_DIR="${SCRIPT_DIR}/terraform" +SCRIPTS_DIR="${SCRIPT_DIR}/scripts" +DBT_DIR="${SCRIPT_DIR}/dbt/nyc_taxi_dbt" +SUPERSET_DIR="${SCRIPT_DIR}/superset" + +# ── Flags ───────────────────────────────────────────────────── +SKIP_SEED=false +SKIP_DBT=false +SKIP_SUPERSET=false +FORCE_FULL_REFRESH=false +ONLY_SUPERSET=false + +for arg in "$@"; do + case $arg in + --skip-seed) SKIP_SEED=true ;; + --skip-dbt) SKIP_DBT=true ;; + --skip-superset) SKIP_SUPERSET=true ;; + --only-superset) ONLY_SUPERSET=true; SKIP_SEED=true; SKIP_DBT=true ;; + --full-refresh) FORCE_FULL_REFRESH=true ;; + --help) + echo "Usage: ./setup.sh [--skip-seed] [--skip-dbt] [--skip-superset] [--only-superset] [--full-refresh]" + echo "" + echo "Flags:" + echo " --skip-seed Skip data seeding (~12 min saved, tables stay empty)" + echo " --skip-dbt Skip dbt pipeline run" + echo " --skip-superset Skip Superset + trip producer startup" + echo " --only-superset Start ONLY Superset (skip Terraform, seed, dbt)" + echo " --full-refresh Force dbt full-refresh even if FACT_TRIPS exists" + exit 0 ;; + *) echo "Unknown flag: $arg (use --help)"; exit 1 ;; + esac +done + +# Initialize TRIPS_COUNT so it's never unbound +TRIPS_COUNT=0 + +# ── Step counter (adjusts total based on active steps) ──────── +STEP=0 +TOTAL_STEPS=5 +[[ "${ONLY_SUPERSET}" == "true" ]] && TOTAL_STEPS=2 # Only Superset + Validate +[[ "${SKIP_SEED}" == "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS - 1)) +[[ "${SKIP_DBT}" == "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS - 1)) +[[ "${SKIP_SUPERSET}" == "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS - 1)) + +# ── Helpers ─────────────────────────────────────────────────── +BOLD="\033[1m"; RESET="\033[0m" +BLUE="\033[1;34m"; GREEN="\033[1;32m"; YELLOW="\033[1;33m"; RED="\033[1;31m" + +log() { STEP=$((STEP + 1)); echo -e "\n${BLUE}┌─ Step ${STEP}/${TOTAL_STEPS}: $*${RESET}"; _STEP_START=$(date +%s); _CURRENT_STEP="$*"; } +ok() { local s=$(( $(date +%s) - ${_STEP_START:-$(date +%s)} )); echo -e "${GREEN}└─ ✓ $* ${RESET}(${s}s)"; } +info() { echo -e " ${BOLD}·${RESET} $*"; } +warn() { echo -e "${YELLOW} ⚠ $*${RESET}"; } +die() { echo -e "\n${RED}${BOLD}✗ ERROR: $*${RESET}\n"; exit 1; } + +_CURRENT_STEP="initializing" +_STEP_START=$(date +%s) +SETUP_START=$(date +%s) + +# ── Error trap ──────────────────────────────────────────────── +on_error() { + local line="$1" + echo -e "\n${RED}${BOLD}╔══════════════════════════════════════════════════════════════╗${RESET}" + echo -e "${RED}${BOLD}║ Setup FAILED ║${RESET}" + echo -e "${RED}${BOLD}║ Step: ${_CURRENT_STEP}${RESET}" + echo -e "${RED}${BOLD}║ Line: ${line} ║${RESET}" + echo -e "${RED}${BOLD}╚══════════════════════════════════════════════════════════════╝${RESET}" + echo "" + + case "${_CURRENT_STEP}" in + *"Terraform"*) + echo " Troubleshooting:" + echo " • Verify credentials: SNOWFLAKE_ORG, SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PASSWORD" + echo " • Test connection: snowsql -a "${SNOWFLAKE_ORG:-ORG}-${SNOWFLAKE_ACCOUNT:-ACCOUNT}" -u "${SNOWFLAKE_USER:-USER}"" + echo " • Run manually: cd terraform && terraform plan" + ;; + *"tables"*|*"seed"*|*"stream"*|*"view"*) + echo " Troubleshooting:" + echo " • Test snowsql: snowsql -a ${SNOWFLAKE_ORG:-ORG}-${SNOWFLAKE_ACCOUNT:-ACCOUNT} -u ${SNOWFLAKE_USER:-USER}" + echo " • Ensure SYSADMIN role is granted to your user" + echo " • Check the failing SQL file in scripts/ for syntax issues" + ;; + *"dbt"*) + echo " Troubleshooting:" + echo " • Run dbt debug: cd dbt/nyc_taxi_dbt && dbt debug" + echo " • Check profiles.yml: cat ~/.dbt/profiles.yml" + echo " • Re-run skipping seed: source .env && ./setup.sh --skip-seed" + ;; + *"Superset"*) + echo " Troubleshooting:" + echo " • Check Docker is running: docker info" + echo " • Check container logs: docker logs nyc_taxi_superset" + echo " • Re-run skipping Superset: source .env && ./setup.sh --skip-seed --skip-superset" + ;; + *) + echo " Re-run: source .env && ./setup.sh --skip-seed" + ;; + esac + echo "" +} +trap 'on_error $LINENO' ERR + +# ── Snowflake helper ────────────────────────────────────────── +snowsql_exec() { + local label="$1" file="$2" role="${3:-SYSADMIN}" + info "Running ${label}..." + SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename "${role}" \ + -f "${file}" \ + --option output_format=plain \ + --option friendly=false +} + +# ── 0. Prerequisites ────────────────────────────────────────── +echo -e "\n${BLUE}${BOLD}NYC Taxi Snowflake Migration Lab — Setup${RESET}" +echo "──────────────────────────────────────────" + +info "Checking prerequisites..." + +# snowsql on macOS is often installed as a shell alias pointing to: +# /Applications/SnowSQL.app/Contents/MacOS/snowsql +# command -v doesn't resolve aliases, so we check the known path as a fallback. +SNOWSQL_CMD="" +if command -v snowsql >/dev/null 2>&1; then SNOWSQL_CMD="snowsql" +elif [[ -x "/Applications/SnowSQL.app/Contents/MacOS/snowsql" ]]; then SNOWSQL_CMD="/Applications/SnowSQL.app/Contents/MacOS/snowsql" +fi + +# Prefer the project-local venv dbt (Python 3.13) over any system dbt. +# The system dbt may be installed under Python 3.14 which has mashumaro incompatibilities. +DBT_CMD="" +if [[ -x "${SCRIPT_DIR}/.venv/bin/dbt" ]]; then DBT_CMD="${SCRIPT_DIR}/.venv/bin/dbt" +elif command -v dbt >/dev/null 2>&1; then DBT_CMD="dbt" +fi + +MISSING_TOOLS=() +command -v terraform >/dev/null 2>&1 || MISSING_TOOLS+=("terraform → https://developer.hashicorp.com/terraform/downloads") +[[ -z "${SNOWSQL_CMD}" ]] && MISSING_TOOLS+=("snowsql → https://docs.snowflake.com/en/user-guide/snowsql-install-config") +[[ -z "${DBT_CMD}" ]] && MISSING_TOOLS+=("dbt → pip install dbt-snowflake") +if [[ ${#MISSING_TOOLS[@]} -gt 0 ]]; then + echo -e "${RED}${BOLD}✗ Missing required tools:${RESET}" + for t in "${MISSING_TOOLS[@]}"; do echo " $t"; done + exit 1 +fi +if ! command -v docker >/dev/null 2>&1; then + warn "docker not found — Superset and trip producer will be skipped" + SKIP_SUPERSET=true +elif ! docker info >/dev/null 2>&1; then + warn "Docker daemon is not running — Superset and trip producer will be skipped" + warn "Start Docker Desktop / OrbStack, then re-run: ./setup.sh --skip-seed --skip-dbt" + SKIP_SUPERSET=true +fi +info "Tools: terraform $(terraform version -json 2>/dev/null | python3 -c 'import sys,json; print(json.load(sys.stdin).get("terraform_version","?"))' 2>/dev/null || terraform version | head -1 | grep -o '[0-9]*\.[0-9]*\.[0-9]*' | head -1) · snowsql (${SNOWSQL_CMD}) · dbt $("${DBT_CMD}" --version 2>&1 | grep 'installed' | grep -o '[0-9]*\.[0-9]*\.[0-9]*' | head -1) (${DBT_CMD})" + +info "Checking environment variables..." +MISSING_VARS=() +[[ -z "${SNOWFLAKE_ORG:-}" ]] && MISSING_VARS+=("SNOWFLAKE_ORG") +[[ -z "${SNOWFLAKE_ACCOUNT:-}" ]] && MISSING_VARS+=("SNOWFLAKE_ACCOUNT") +[[ -z "${SNOWFLAKE_USER:-}" ]] && MISSING_VARS+=("SNOWFLAKE_USER") +[[ -z "${SNOWFLAKE_PASSWORD:-}" ]] && MISSING_VARS+=("SNOWFLAKE_PASSWORD") +if [[ ${#MISSING_VARS[@]} -gt 0 ]]; then + die "Missing environment variables: ${MISSING_VARS[*]}\n Copy .env.example → .env, fill in values, then: source .env && ./setup.sh" +fi +info "Account: ${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT} · User: ${SNOWFLAKE_USER}" + +echo "" +info "Active steps: Terraform | SQL scripts$([ "${SKIP_SEED}" = "true" ] && echo ' (--skip-seed)') | $([ "${SKIP_DBT}" = "true" ] && echo "dbt skipped" || echo "dbt") | $([ "${SKIP_SUPERSET}" = "true" ] && echo "Superset skipped" || echo "Superset") | Validate" +echo "" + +# ── Step 1: Terraform ───────────────────────────────────────── +_CURRENT_STEP="Terraform — provisioning Snowflake infrastructure" +log "Terraform — provisioning Snowflake infrastructure" + +cd "${TERRAFORM_DIR}" +cat > terraform.tfvars <<-EOF +snowflake_org = "${SNOWFLAKE_ORG}" +snowflake_account = "${SNOWFLAKE_ACCOUNT}" +snowflake_user = "${SNOWFLAKE_USER}" +snowflake_password = "${SNOWFLAKE_PASSWORD}" +environment = "${LAB_ENVIRONMENT:-lab}" +lab_cohort = "${LAB_COHORT:-fy27-q1}" +EOF + +info "terraform init..." +terraform init -upgrade -input=false -no-color 2>&1 | grep -E "^(Terraform|Initializing|Upgrading| -)" || true + +info "terraform validate..." +terraform validate -no-color + +info "terraform plan..." +terraform plan -out=tfplan -input=false -no-color 2>&1 | tail -5 + +info "terraform apply..." +terraform apply -input=false -no-color tfplan 2>&1 | grep -E "^(Apply| \+| ~| -|Plan:|Changes to|No changes)" || true + +ok "Snowflake infrastructure provisioned" +cd "${SCRIPT_DIR}" + +# ── Step 2: SQL scripts ─────────────────────────────────────── +_CURRENT_STEP="SQL scripts — creating tables and loading data" +log "SQL scripts — creating tables and loading data" + +snowsql_exec "creating tables (01_create_tables.sql)" "${SCRIPTS_DIR}/01_create_tables.sql" +info "Tables created." + +# TRIPS_RAW now exists — apply the Terraform resources that depend on it +# (CDC stream, 5-min CDC task, hourly agg task). +cd "${TERRAFORM_DIR}" +info "Terraform — creating CDC stream and scheduled tasks..." +terraform apply \ + -target=snowflake_execute.trips_cdc_stream \ + -target=snowflake_execute.cdc_consume_task \ + -target=snowflake_execute.hourly_agg_task \ + -input=false -auto-approve -no-color 2>&1 \ + | grep -E "^(Apply| \+| ~| -|Plan:|Changes to|No changes)" || true +info "CDC stream and tasks created (both SUSPENDED — resuming CDC task now)." +info "Resuming CDC task (CDC_CONSUME_TASK)..." +SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + -q "ALTER TASK NYC_TAXI_DB.RAW.CDC_CONSUME_TASK RESUME;" \ + --option output_format=plain --option friendly=false >/dev/null 2>&1 || warn "Failed to resume CDC task" +cd "${SCRIPT_DIR}" + +if [[ "${SKIP_SEED}" == "true" ]]; then + warn "Seed skipped (--skip-seed flag). TRIPS_RAW will be empty until you run scripts/02_seed_data.sql." +else + # Auto-detect: skip seeding if TRIPS_RAW already has data (idempotent re-runs) + TRIPS_COUNT=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + -q "SELECT COUNT(*) FROM NYC_TAXI_DB.RAW.TRIPS_RAW" \ + --option output_format=plain --option friendly=false 2>/dev/null \ + | grep -E '^ *[0-9]+ *$' | head -1 | tr -d ' ' || echo "0") + TRIPS_COUNT="${TRIPS_COUNT//[^0-9]/}" + + if [[ "${TRIPS_COUNT:-0}" -gt 0 ]]; then + info "TRIPS_RAW already has ${TRIPS_COUNT} rows — skipping seed." + else + info "Generating 50M synthetic NYC Taxi trips — this takes 8–12 minutes..." + info "Dataset: Synthetic TABLE(GENERATOR) with realistic TLC distributions (2019-2022)" + info "Zones: Real 265 NYC TLC zones | Payment types: realistic distribution" + info "You can monitor progress in the Snowflake UI: Admin → Query History" + + # Run seed; if it fails, check Snowflake error + if ! snowsql_exec "seeding data (02_seed_data.sql)" "${SCRIPTS_DIR}/02_seed_data.sql"; then + die "Seed failed. Check the SQL script for errors:\n + • Review ${SCRIPTS_DIR}/02_seed_data.sql\n + • Verify warehouse TRANSFORM_WH is running: snowsql -a \${SNOWFLAKE_ORG}-\${SNOWFLAKE_ACCOUNT} -u \${SNOWFLAKE_USER} -q 'SHOW WAREHOUSES'\n + • Check Snowflake query history for detailed error messages" + fi + + # Validate that seed actually loaded data + TRIPS_AFTER=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + -q "SELECT COUNT(*) FROM NYC_TAXI_DB.RAW.TRIPS_RAW" \ + --option output_format=plain --option friendly=false 2>/dev/null \ + | grep -E '^ *[0-9]+ *$' | head -1 | tr -d ' ' || echo "0") + TRIPS_AFTER="${TRIPS_AFTER//[^0-9]/}" + + if [[ "${TRIPS_AFTER:-0}" -eq 0 ]]; then + die "Seed completed but no rows loaded into TRIPS_RAW. Check Snowflake logs and network connectivity." + fi + + info "Data loaded (${TRIPS_AFTER} rows). TRIP_METADATA VARIANT populated synthetically (lab telemetry field)." + TRIPS_COUNT="${TRIPS_AFTER}" + fi +fi + +ok "Tables and seed data ready (${TRIPS_COUNT} rows in TRIPS_RAW)" + +# ── Step 3: dbt ─────────────────────────────────────────────── +if [[ "${SKIP_DBT}" == "false" ]]; then + _CURRENT_STEP="dbt — building transformation pipeline" + log "dbt — building transformation pipeline" + cd "${DBT_DIR}" + + export DBT_PROFILES_DIR="${DBT_PROFILES_DIR:-${DBT_DIR}}" + + if [[ ! -f "${DBT_PROFILES_DIR}/profiles.yml" ]]; then + if [[ -f "${DBT_DIR}/profiles.yml.example" ]]; then + cp "${DBT_DIR}/profiles.yml.example" "${DBT_PROFILES_DIR}/profiles.yml" + warn "profiles.yml not found — copied example to ${DBT_PROFILES_DIR}/profiles.yml" + warn "Edit it with your Snowflake credentials, then re-run: ./setup.sh --skip-seed" + warn "Skipping dbt for now." + cd "${SCRIPT_DIR}" + else + die "No profiles.yml found.\n Copy dbt/nyc_taxi_dbt/profiles.yml.example to ~/.dbt/profiles.yml and fill in your credentials." + fi + else + info "Installing dbt packages (dbt deps)..." + "${DBT_CMD}" deps + + info "Running dbt seeds (reference CSVs)..." + if ! "${DBT_CMD}" seed --full-refresh; then + warn "dbt seed had non-fatal errors (no CSV seeds present is expected)." + fi + + # First run: full-refresh to build all models from scratch. + # Re-runs: incremental to preserve producer-inserted trips. + FACT_EXISTS=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename ANALYST_ROLE \ + -q "SELECT COUNT(*) FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS" \ + --option output_format=plain --option friendly=false 2>/dev/null \ + | grep -E '^ *[0-9]+ *$' | head -1 | tr -d ' ' || echo "0") + FACT_EXISTS="${FACT_EXISTS//[^0-9]/}" # strip any non-numeric chars + + if [[ "${FORCE_FULL_REFRESH}" == "true" ]] || [[ "${FACT_EXISTS:-0}" -eq 0 ]]; then + info "First run — building all models with --full-refresh..." + "${DBT_CMD}" run --full-refresh + else + info "FACT_TRIPS has ${FACT_EXISTS} rows — running incremental (producer data preserved)..." + "${DBT_CMD}" run + fi + + info "Running dbt tests..." + if ! "${DBT_CMD}" test; then + warn "Some dbt tests failed — check output above. The environment is still usable." + warn "Tests often fail on an empty dataset (before seeding). Re-run after seeding." + fi + + info "Creating secure view over ANALYTICS schema (04_create_secure_view.sql)..." + snowsql_exec "creating secure view (04_create_secure_view.sql)" "${SCRIPTS_DIR}/04_create_secure_view.sql" SYSADMIN + + info "Resuming hourly aggregation task (HOURLY_AGG_TASK)..." + SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + -q "ALTER TASK NYC_TAXI_DB.STAGING.HOURLY_AGG_TASK RESUME;" \ + --option output_format=plain --option friendly=false >/dev/null 2>&1 || warn "Failed to resume hourly task" + + ok "dbt pipeline complete" + cd "${SCRIPT_DIR}" + fi +fi + +# ── Step 4: Superset + producer ─────────────────────────────── +DOCKER_AVAILABLE=true +SUPERSET_STARTED=false + +if [[ "${SKIP_SUPERSET}" == "false" ]]; then + _CURRENT_STEP="Superset + trip producer — starting Docker services" + + # Check if Docker is available + if ! command -v docker >/dev/null 2>&1; then + warn "Docker not found in PATH (--skip-superset flag would avoid this step)." + DOCKER_AVAILABLE=false + elif ! docker info >/dev/null 2>&1; then + warn "Docker daemon is not running (docker info failed)." + warn "Start Docker and re-run: ./setup.sh --skip-seed (to skip data load)" + DOCKER_AVAILABLE=false + fi + + if [[ "${DOCKER_AVAILABLE}" == "true" ]]; then + log "Superset + trip producer — starting Docker services" + cd "${SUPERSET_DIR}" + + info "Starting containers (docker-compose up -d)..." + if ! docker-compose --env-file ../.env up -d 2>&1 | tee /tmp/docker-compose.log; then + warn "docker-compose failed — check the error above." + warn "Superset will not be available. You can start it manually later:" + warn " cd superset && docker-compose --env-file ../.env up -d" + DOCKER_AVAILABLE=false + else + SUPERSET_STARTED=true + info "Waiting for Superset to be healthy (up to 2 minutes)..." + SUPERSET_READY=false + for elapsed in $(seq 5 5 120); do + if curl -sf http://localhost:8088/health >/dev/null 2>&1; then + SUPERSET_READY=true + info "Superset healthy after ${elapsed}s." + break + fi + printf "\r waiting... %3ds / 120s" "${elapsed}" + sleep 5 + done + echo "" # clear the \r line + + if [[ "${SUPERSET_READY}" == "true" ]]; then + info "Registering database connections and importing dashboards..." + bash "${SUPERSET_DIR}/init_superset.sh" \ + || warn "Superset init completed with warnings — check http://localhost:8088" + else + warn "Superset did not become healthy within 2 minutes." + warn "Check container logs: docker logs nyc_taxi_superset" + warn "You can retry Superset init manually: bash superset/init_superset.sh" + fi + + PRODUCER_RUNNING=$(docker ps --filter "name=nyc_taxi_producer" --filter "status=running" -q) + if [[ -n "${PRODUCER_RUNNING}" ]]; then + info "Trip producer is running ($(docker inspect --format='{{.Config.Env}}' nyc_taxi_producer | grep -o 'TRIPS_PER_MINUTE=[^ ]*' || echo '~60 trips/min'))." + info "Monitor: docker logs -f nyc_taxi_producer" + else + warn "Trip producer container is not running. Check: docker logs nyc_taxi_producer" + fi + + ok "Docker services started" + cd "${SCRIPT_DIR}" + fi + fi + + if [[ "${DOCKER_AVAILABLE}" == "false" ]]; then + warn "Skipping Superset (Docker not available or --skip-superset flag set)." + warn "Run manually later: cd superset && docker-compose up -d" + fi +else + warn "Superset startup skipped (--skip-superset flag)." +fi + +# ── Step 5: Validate ────────────────────────────────────────── +_CURRENT_STEP="validation — checking environment health" +log "Validation — checking environment health" + +VALIDATION_OUTPUT=$(SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + -f "${SCRIPTS_DIR}/validate_environment.sql" \ + --option output_format=plain \ + --option friendly=false 2>&1) + +echo "${VALIDATION_OUTPUT}" + +FAIL_COUNT=$(echo "${VALIDATION_OUTPUT}" | grep -cE "\sFAIL(\s|$)" || true) +if [[ "${FAIL_COUNT}" -gt 0 ]]; then + warn "${FAIL_COUNT} validation check(s) failed — see FAIL rows above." + warn "Common causes: dbt not yet run, seed still in progress, or empty tables with --skip-seed." + ok "Validation complete (with warnings)" +else + ok "All validation checks passed" +fi + +# ── Summary ─────────────────────────────────────────────────── +TOTAL_ELAPSED=$(( $(date +%s) - SETUP_START )) +TOTAL_MIN=$(( TOTAL_ELAPSED / 60 )) +TOTAL_SEC=$(( TOTAL_ELAPSED % 60 )) + +echo "" +echo -e "${GREEN}${BOLD}╔══════════════════════════════════════════════════════════════╗${RESET}" +echo -e "${GREEN}${BOLD}║ NYC Taxi Snowflake Lab — Setup Complete! ✓ ║${RESET}" +echo -e "${GREEN}${BOLD}╚══════════════════════════════════════════════════════════════╝${RESET}" +echo "" +echo " Snowflake: ${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT} · NYC_TAXI_DB" +echo " Warehouses: TRANSFORM_WH (SMALL) · ANALYTICS_WH (MEDIUM)" +echo " Schemas: RAW · STAGING · ANALYTICS" +echo "" +echo " Completed in: ${TOTAL_MIN}m ${TOTAL_SEC}s" +echo "" +echo " What was set up:" +echo " ✓ Terraform: warehouses, database, schemas, roles, resource monitor, CDC stream + tasks (SUSPENDED)" +echo " ✓ SQL: TRIPS_RAW + dimension tables" +if [[ "${SKIP_SEED}" == "false" ]]; then + if [[ "${TRIPS_COUNT:-0}" -gt 1000000 ]]; then + echo " ✓ Data: ~50M synthetic trips (realistic TLC distributions) + TRIP_METADATA VARIANT" + else + echo " ⚠ Data: TRIPS_RAW not seeded — re-run setup to load data (~12 min)" + fi +fi +[[ "${SKIP_DBT}" == "false" ]] && echo " ✓ dbt: FACT_TRIPS + DIM_* + AGG_HOURLY_ZONE_TRIPS" + +# Show Docker/Superset status +if [[ "${SKIP_SUPERSET}" == "true" ]]; then + echo " ⊘ Superset: skipped (--skip-superset flag)" +elif [[ "${DOCKER_AVAILABLE}" == "false" ]]; then + echo " ⚠ Superset: NOT STARTED — Docker daemon is not running" + echo " Start Docker, then: cd superset && docker-compose up -d" +elif [[ "${SUPERSET_STARTED}" == "true" ]]; then + echo " ✓ Superset: http://localhost:8088 (admin / admin)" + echo " ✓ Producer: ~${TRIPS_PER_MINUTE:-60} fake trips/min → TRIPS_RAW (docker logs -f nyc_taxi_producer)" +else + echo " ⚠ Superset: startup skipped (check logs above)" +fi + +echo "" +echo " Next steps:" +echo " 1. Open the Snowflake UI and explore NYC_TAXI_DB" +echo " 2. Run the 7 queries in queries/ — understand each migration challenge" +echo " 3. Review dbt models in dbt/nyc_taxi_dbt/models/" +echo " 4. Complete the worksheets in 02-plan-and-design/worksheets/" +echo " 5. Get SA sign-off, then proceed to 02-migrate-to-clickhouse/" +echo "" +echo " Cost: ~\$47/partner/day (auto-suspend = \$0 when idle)" +echo " Tear down: ./teardown.sh" +echo "" diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/Dockerfile b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/Dockerfile new file mode 100644 index 0000000..a2cc429 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/Dockerfile @@ -0,0 +1,13 @@ +FROM apache/superset:3.1.0 + +USER root + +# Install database drivers: +# - snowflake-sqlalchemy: Snowflake connection (Act 1) +# - clickhouse-connect: ClickHouse connection (Act 2) +RUN pip install --no-cache-dir \ + snowflake-sqlalchemy==1.6.1 \ + snowflake-connector-python==3.7.0 \ + clickhouse-connect==0.7.0 + +USER superset diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/01_operations_command_center.zip b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/01_operations_command_center.zip new file mode 100644 index 0000000000000000000000000000000000000000..79d8c239eb9457d5264420b41d3a9ef81ddeef31 GIT binary patch literal 15388 zcmb_@1#nzRvaQfUixBxLD$7pTi(jr*~miI+)!KI+R;wmkj7Ql(n2$OS*nW{F7W&b36nNl3ked*Uot#V zEZ$#$Up!y7Hl<$PbmeTzDoG&x+zOu`$e#IXH0!~bBmhBgVgLk5WN?<~hbZ`R`uDWL zX&X_z9g^*ffE|)(VPY3|!$WHrE>*p1ec|2sgS^1GW5UYuy-por)FKzG&(n*w+>u=C zS!1NStv_^f8qjKryjIg(^2o3rJN;s@&S4GVY8D-s8rOWf3YZ>o&lOK@dd0GT7#V71 zGZ7Pa*;s8l5tk9#b(Ze3^~A>yWrL;h-QgG@$ND$3K2EOeOy#br#o+e;l(xYmxs#EGUi#85J3x&Dbt z{|Ch(TQCgMq!qIp_Ojz@1$NPD;;OkbLead_=2*yAW{M1PKGD3r8LVU2KM(pG;r6-Y z`xyOv)BG=EWohW3`*Yxb8xSW$J9|@WD-Jw*8af&}NC#INLk>K_e@;6{2UAN!dk0-h z8xB0upL36zj*uPCe<)CeEV`1v>TX&Dj z4_htM!F@lb4A#;^mBs6iM8q@!oRyUW_UepPN2B8{FIMD_&sB)A%k}as3DP7zq)xgJ zgh^`QWEB6H!%KnRNiWJM#7Kr1pIRzmV@cdW98o7lCf=JozzAb`&# zgla5c6uZHd*q455Gi5t+=?Q7JPAb^F}h#Js$LIXVs0J z^sFnx((kaRz0sZ&@Z;QcLVFxfySsUmLHY{?17xbl3%}KBhrP8mr!joI@xG+g(_^{> z%kj4cI@%)paoClI(&^K)ofo$z)fj8_sV4)&Xn|Zc2quygfQV~Q4Er4VB1q60vFV{~ z6N`IfliuP*!c{-A36%u>o)p*hPd&yN0xe0VnLZodRSXp(`Jy;}GA3VKeB~zV2PW6M z9c69gDX~6&*5$D~p#;NrpTN~z(q#%h#XJzwtqj7)^jGo~wkzyQluWhc6#51x9Lv3@ zTMAR_crmI`Hw3$iP6iuehtrn}6wLFFK`gk+v_xfqI;rQ?jv>*3%c~ZLAND{dafrHug|l;Z zSNGforU{^ix4Zk2hJFw_l#cLyc`J@hc%)BIgvhVFW>45g-b`r8m`7~UY|W~)_8QrG z)km!O#QYWPWYwW4EXt^2b2|S>lMOXe(GkgabN=}#$M2wncuZ#3R(twfP6Yccd|!N&*7_n7=PG_5Z^eZ}JZH7OvWMhE9f7j=w|xuA-#% zHVa(qu`--{wWdl*ESZ!=HYk#+sQ))Z0bF$|9iMD+4io~}QW5rJl$UdCvAX$1$*Q1M zkgrlR(N{)mY3OpXj$?GBplR7P2rlX@j&1$S%k*n>d3fmkCGntzhM)Mp584wN`p&5m z*T&*fV@~ZLFlH(l_7$)srA6pm=%dF^Psw`qTfhI#T@cGAi&@xg$Uy}OY!?`k! zM<`=;vO4O)_Uj@vzGM!SvkEE-XQe_Svh!AYN%_x+ez!V6jO#|$MhE{o6M3tV{igkKawUx-!3--w7}e{5FcL}>chtzmd* zn2$UOD-Y4Ss-Y3deCeI(VFCra_&5m4DDo}7kBNc2rIF{DFWMDC9tdTjX0T4jI@uD=`d*_sA}~e_zmmzn?1lC^UsZ-NCSCd&lIGK*ob3I z_?YGh9W=kK;bOP=mT0l1N)dC4T!m`PYFUX_@Y6j<*fi_EzV0)RRk!!xhmUw?S$+@B zK!<6S){1_HJ@`Zw3%5>k_L=q-Uo~S?U1K((^la=-!uhZTH;%YXv%}Hz?P7X2Vj}_K zY@Jc}272oj=Ebze>!OVVeZhfu-*_&?u-Ib`WWjNKZa{B@0CnU$7jBfqa{cC_m^IDl z!qCr|)@9V`F5MZRFP%Hzjzs1Sl!V*&t8F#WRI@24L#os(oQ zee`jB!^@R?D}twfAp2+W!(A-=ATciT9wohhapB1Hz6U2j?nSBu2|=JbN4snl-G`+v zv>gJ~W!s<%LkvORZ%{3ba9QiW^ax!amwv+-?PwjSOG>7I2c2@u(U#{c@Yf%c2+kU# zA=xLaonNYT;)?Re%ov9|mfzc_f2cY-H!QEuP2JPbecmBXmmDX#ci?JI-(A|jsrkWh zk?Mj@FBC9lZ>#lUpYnAmcjE5Dnd%n7r*(A%64MfhgwW%zYrd~H^f?v_c#^ZeRg3JD zW7+GMp(IE@&fHHmnsf9z3%acgpSa;gLEEW*W&|-ltQMm6;_Y``pYMQm>&Qvnn<+ZK}a-(!^ofu1lUNi9aVe`q{KLNrGKg zLVUeE*};eb6v>ITI3d<%6xf0?X}uJXFa-ov`%`2g3Iv1t$^^gD75hZBWXaf)t3PpuUTcRWQI!eVE_C&yKDF{esol3I_znDvF=2oq*Tp|A63m zemQtS(xhcW(8cAR5lAcoD2;S-(W)F(3_>zlvEMS zkYd5tS;C<*{Y-p{1%(n|GlERtVl9AZt6(kHLCmvmLFsu(0~mZqH1%-kIUr&`PxBhU z#x6>(^~#b7zN34QREAZ7k+v>qGP`6xmtv_WxclKrBfZKb?2)BrLSz2VH{TcWwY?Rp zjFhvM_!2=V*GGqQsoVl@`9cE)_|}RG#HQzGLhXX(~zC(|VXG-*8TxSgb||A7)U% zs}1{fN@(i7XT(uR!OFz$!ckS#&FZA1`!+ay*U<$-bwaG7VyIz2yY$sG26-+}$O%i= zNNA%AI_!Gzq}ZfCyh|$7Ns?lSZ*s09w+S^1)ZU>x*YjG(T)|*>T(#?r5B{E#ROr#> zn`q4P@du$ZPje}QML~8FUlGpv`Tb9aW! zCQ;MG!v}NkrW4r5>*87MiK^MW)x0?ZG$*z0ZKs7}+#7f9C2QW-6NB2T`j`4uIr}zk%@z zcBVh$7(+X4k@uaQHvK;UUEF$^4X*7;8NAs6waSmpc1opz=z@jsO08cQ3JH);ylzwp zK{3`Y_QS(Dr+{2MRny^_YnlM%NA?}AwvTH^7JB1 zsIEvmhN-uDB+S}&iP61R^<;&O5kwO5nG_B6n>@i@G#dsel_fKwCYkiTM8=`{T*j@w zVyPv?s2~ut3Y}l+PjC=&4+#wMUT@&}vcsXc0EJxK39du@Kv5g6$l24fywgW`L)8q3 z$;U$;GpEQJ(>~HZTjl!C_?NOl)={a7TUi5V;fZi2$;9nKj4NyL=&Su$aA5&IFe6?J zHM`jI^7@#I7c8V^2ny!{n>w5Ji&UOHj@a2K1sJ!5@j z#f-IEX)F_RFy1UIiECCu&**#BeM4+6)x}yMbgiuhH|5r8m@ly;a+xy1*IG;6%kudH zlAbWCfRW-ghA!d#sVARvgI>DPE4k7E%=lH!Oz-sdnA+vT(`R74G!k=o#5AU$9=3u4UrZiN zi9$rM9_TO`D0EQPu7Z1I9H;|$B}0~`ww8=ct(e2ZQLvWcpihG#6>~kyu z3$D?D5jri-|}lszzQoZC4#6C{AdwD*Yq zm8YW+B-!{!=9^AEn%Oz&$+ZvGNO)iV;g_-4?Rc%Yh>`MVkng-7wSL zON?-YlD&ZT?$&YM<4R328ok*sck@Nl=uIiwS(1exRXjd1d#+_1I-ul^Ac8q56TN=n zHWeK$DI6E>947{AnRk&63Yu&1$-HCpcujG;dv0$2N7UY zCya5@oAM#ly!p!>==sHSqY=VaL10tad^UpcTplnK4P^8un@eCEwJhT#_aEzFS}3el@pDpXUM)qR{j`e&mckp58qfJwO-jd~g^8Y}53%2vxaTM}k~8N%U`UM3t5j*xTgq2HKEazBS-2EMj(f|Z$L z{h*1^9eqF*e)37{J1ApXk{=(!KgmehuRJFiy7u;~-1~k7so17IJ+{%3>R9!BTXmc} zga&Ct#|TXLu*-_`dgpSG#)$)flF|H!z++lIQ>R(2(S_zL+|}M`!V?*5x0T|?0P1{?Vre} zQAI`f8M_q@Iu9msvsDcc^{!l7(q63P?Rw;y2x^7A*<< zG7_WQa<6X`;?LA47dkFTxKS`*LFe1MI&~i2Txe0S2R<~Px4Y>%myE@i{BVrC*U-;S z&rk0plJrs+5Vu~4aImp(VfCCYyy!TLNd1Jq^I(L>$#P*tO2Zt&AHM+q+2wJX^PTv- zhc)DI001I?$K_c&=vrvY{hqPr@%2{^|EfM9!0Wkjedfw^m36{seyt>WBMqQDyumDz8#C|v_5gPwf^`> zi6k0hbepx)l6?60=9{NfO`A6E}L8#Po`^&L=`}ZnLGx7C> zHG=UiLHm%)A6V2-TCw|bNbH|hvwLIqq04G>zlCHn>W6G$O65B)k?fhb@=B-dIgl6( zIShkiX7f~72~U3r7<28Tzm;mwr)h%rF9n#FsnTNzt7Vr=@d)iHJtRmQC(i2bj;?sB z#^-Z;@tj=9K1iFLchKvcDdIPc(W>?#m6Xzh$bQLAh!*HZCPAL>(+eX$qIsefeLGBt zRc2DFRgKA@mckw92u=$CFBuI1bUNoADIsv#$yxir&;~(1%MZl=e0e$wVu_*YT@jtC zw~2NeG(0(jtEq*vaejTgz=M~f64PBVjp>Asb1VS1MW}>ZZqNk@O5*XfCH?G7h)(&Y zDn1u3S6=Z@D(b?z!~((^v7id+7?l7{lX=Z9ePBZ^EuAX@vq%VD5}+bW?TRu=EJ9!~#RG5PI`G62TCs=GreOE^K++*JY;)ocP+$Xx$41z?s zPLwY$BRYb7`RYkb%r=Ui3&#KPI)(3r z4u_SHuhr~~i&&I}xCoq2tVu!Z>83b}wNp{Z0x!XTW)c>R$hYH|S#R;naVr$eM~Ce3 zQUdoL^kD|b^tlu#O4>Jqb0k9tBjA*v&tR`n^-10QPLvh*Cx~{-K+q%*B^mV<% z*ATs2h9nL`eHjC|hT$6+H+96fOAV0|+T=2dtpJH)%|NkeO!>Q*qd?t~CYjVdIe#=T z2UIxIpr8O(9NK^iP5J~hV|eetFzPBECaB{acx%EMQVy+~$Y>HaDne@=X(A!1=>;?a z^N0f?o8`h$ghFPpYOrw4=}M&_!dZ@B$zlz#FU^Z!U##vrGag@5$E!XbK>={|QtvyJPDDovTp#V!d9fbeWolF_*x^3nZW;dK?SjB`t*ydjWO0u-3V8HrXNQh1(C>IcFER zXl7xoNL!qeb_1<4dIRm!m=$B^xMjZ>qz|;v+brQ6)W5JekGF>9 zAKy`4_-~{9@8QkAV?5J8FkW4Hn*+}4yfXN?2Jai0srZ-g$i+^RIUR|)+*7Is>>LWZ;j5|)C88FsoGRjODrxDe@xr|*gK3Xu}} z5>1`6NeLpwaG~OBT+k^I1(QeLiLsACkw`H*YJ?@ z71tzQsbU1GnJ6|G&q0s`gpoANuXoDREyp3c0bFBoGg|go2;6Z@Ap3Q?0T6w#!>XW>vzfPKWIH5GGQz;4L?BoA++O7IoG zjfBAP5P?*XOUPr@A(|@5mu6FSV5WN}8m~6av$`1Ydi2P=xr*QiK~gB`JW;l79HKgq z2~V-Tt%=Ht!De5wIlcetCIq@sr5|qDBv2N#ul(bT^vX_S9$@Vje*FVo@R=tFO9~i| zNpU@VIc|Uh1q}d3+~Mk}g-3+(_PZw$MAW1Yd^xn^$_l5Yg}pw1HsfIDCD+b-M|aLc zekPV67{Q=Q9@W9*9R!K{TSJC4546uDq_kquM&ZfdbB;+iyH%FqR#<$E$=TT5(TT$* z)X7F3he=s+GyM4g80>oIRUlG@*^Eph3`&A!T=dPb5d*YKHjojAf}zr@FG3*I>vDTA zlDBW3_RuoV9*JMH)=`0K|6UojsBm?W;(gtAqxBtJEaQ0giWyk)T}=uD^MyW!0M;2sfV0+vK@ zWPd{Rv1TgS4ZO&u#x!h6dh|1Z@&-PPW*2d+!@?D3?rboZA{43{-(+v_+;d(&x z?lWTfwFK&=hgEW44^2}W<8^>Vz0^h7$)oC=X@wgKmr8oZq2;>Gnu3`)poeDv=3+F2 z-J-r}a4(F!9A8;_Sd)~z ziqI96>RY^oXzbf&uOva^C7~ADO&>nM5=rXnGmwW5_MxCg7m#z97(vMQS<^V^Pr6d% zWrU$EpBC{p&^zFk93ds9+LqXmxD@AQXMmC|<>+2!R2x@NXiGrUFy+$PR0sU%HxlAz z$+Z-$*WXTxveD%ikh_NI6ZAD^)9pjhD4E6qYOtIqAQ6*=bD#NQ&M|vrUaaxza7>=a zlRdYLJ8^p3ApfoXh>0cUG3QPEmPe~bN7r8RNfMVfv5woXrqn;fR<+W^n~j-o=EMZg zD6vkvw_;%&!!>ZX{PEGzpkbatGZy#Mu&~|I(unx%mfDE8vMK4%4dH{34p_N?CQ`@w z2HF%rdvwHU1k@SMN6C#lTP`?@uX-G+5t`*EfXKC4eLk}qeLh7E^a@lI%iQ%^eE|4t z+qZI{=!5!-JXZqzZWMMo-hhi!K2LtbSseHf$}{DkxM<(>cbi`KS+JUMLRO}6uwlmB z1nf?3d}ZJ2Py=2jeH#0TAU|UlE4GZfe?E&@rq{`aRR5TR8PNS|G6QA$md}bcIjsfP z+`XpC1Rb=tc<6&QJ=Xr`9lV=cme=(%!7QboNDm^*M(ma1k8p4W$^g*RQBJue3AK#-*3k7L_RaWv8 zYF$V8o>vjMq|r&=)BBGX^*I)Zop;teT3)HEQ(>0|T<&jxNR3f6#9?x!uc7GW2QNw$ zMPeeS?#{Kx+J)PP56BteuV||y!tE@|GKE0v>QyS2*a6?bR{~XV6gHJI17(}9xvA|n z;%%t&I$YK-7wpI%Voj~e0b=$+3Emw8>v(2=7qd8UFqiO9Zx6^~X?sFoU#QzzdY4Ck zXZB{BdQ}uK<$jukU4c_?7}=bL9+0^5r8PwM(8V+}x{;*}7o}RN3h1IK|JS%ey5%Ee zH%R)|;9rAqO*agQa<|O-^P=aFYiva`3#axIu@4&6 zaVOc;8VWLb{jSyi<7GgmawQy1U1^!4uzVhQ88E&w5qbeWO|y(d4WO^(E-6-BX#V+!X)!=C56T&Tlw z$-+AYs$5lW@RjCvy zRfgjvtV@K#C=}~G2tb>v=(TIv_94q?B?IbR!WU5}CzOZ{!oVaB)d&2+k-b_;UXp=a zHlRf&CnwNwE@yDI_3N2*xGLPDA{|EPMTfz!TM;KL-ZDrKDQb$eE(3%~%`HcobAR-> zpg}<>Pll2Te<-MIY!f7JCZBQ7p_YFr+d?AD2Q~KjZLrxszE& zD_c9}cN@rOuZoY8diVXV~(_r5*l0-GGFzjwz|U^i;4ik+pn;mOXZHCaUM#07>y^ias=!*_!$2TMU<5 zwLHn~p*W=hv1+0jwC0{6JDQj=XQ5n<7)nsvZZ(%wA4M0ZbCu6r>sG6DMDI<2=VcbA zAOU(II0s_4V6n81sC8!%hEI{eKFi-1QNk5#YEsRIYD~*G@2%VMI5ohpqiH->Y%i4v=FV0tJ>A~`ndfyDSd#w`bGr{R|A>XvtMvT_-;duD^+ltiP zem!C%1Vyg#dJcIjvF5QX91=r1HKD`{%eIKmyGc*eo{+2 z0F(B-vKL%7WC+}7HQ2de>K2&`dgLiMgyr(w$;q)trXa&LYyrgtZWrL)*U|hOR`@Rh}u|IocOCE2EKxfWWs>~Od=vb z(BcR5=E%sO!EA= zmw~0yEwLif<%Ppa&5JF7kqXAxKlrBpL7=Fl*xDilE1t_Y7Hkv|8rPy}= zdo)6mb0()iSF32PH*uJIx2J{O^hLDGssBqBQP7hZ_P06{LhV-LJ7p)3{KKR0L9d}K zR@@5bog<^GyT_%~zAu1~9}20-3r%5~8q0->kZC{oc5%kd*KJ^9aWiIYN;2X>`Bhn~ z$zcv^bs%wbFJF|7fFuOip>l_uagsz4p+*GK%3Ys9z;bmQBJP1oNgFLF1j}2@0`rX1 zv9UOb`=PT)vB|TwdUkmQUU;x}vp^X>$6o`Ifzn(+^CU`OSogtOE)T8Pj+zTUZtK`I ztlW3%vv+OGj;;r!9l^6X`uh6;BR+jY+MJod8T($qPaCe}<7P2|6U+>UvlZb_nvN3- zHGs#;#yp*_TgkLJd*@zG>zUSCcYR)|NbG;P-CP$EF$+?32^|$cCQU35?bEtI_wF~eSY!TpU+yR&e7~P*L8|z^G)SBga%OY9T*qIk{ ziKz>MS>&iXDen~(wCExXs7ZYhQs!h(aO9*5ghLg+LBdtFX0RRfM$KHNdxV}&RuSu# zj*Rw1*Qbjh@H6y%%o)0MyX&j?6_)f|0j`aE4OM_}lrwQKqh_K+f;F``n?C>lC^`|qG^zc`eDh7uKe@F(KUVO^8Kkpyl-Jbc)PaBo8Z2n4J$J%X`~&P_|Lnk7|7AsW*~K1ER@{0=}MF zzAshpaAL}^Ts>T+-ViOv@MK8;Ig(@Osf-uZIkIv?dnhjxKgvx4=cZ@m3$RjQuG~Rt zc@i8-DsU@y3Q4c%VvLN@$2tph_ZJ(T5O%E-n7T%b_+$B`LkWLt4YBv1n0F zk~pVOi4;6h_Ytjj(g0o(j9AA@qtC>WaRyyPQ1+7Fs)g~Q_ywxz@zCh+jYwmVg5*TyYH9XDo&$%l)@9DkqQGArRZjl5m8YbL

E7!{;_wy5M19kf7JHz9qODSc?Nl^?Xm>uQ2(nV!@ z$m4Q?hglp-B)eqlkuchL^)!38sp?};>Ti}~r?U&4xf(_WM z1<9RB_ibPwr_FAD3&aW~ub~=#Gg?j#tXrphfu<;D?O>zUq90cM%Sn$d`EZaBFLtm+ z907;G{jF~M($*y1I%k}5i{G~0;~rc?<^o*jg&&epi$F}d5cBI%ceLs)`|pc@m&9h| z38wCn&-WkRzH>*BUdQX~wCdcB-K{SMnjs6PCLHNuO;AN_J=L)HM3l9yoG16FJ6oh0 z3(>MpA2U1t(3hreI;+&Mw4|iIZpK>BTzNt#rxhFX z>>fUYEqs)vm!Cc+vaXx|_EC8JP?6ru18m$#wYpU#({L-bqoHZOX@Ps_`3nhgnO4rJ z=QhA>q|xWU^WbQT+;ff_2C zZ7ktj%62<_Hfg=B0~FR@1J(~vl&z>W)DxFXGX{LxEzQCP#@FuMVe zUSrG6HJmjT%`T&e_f2nRybWjCM-Q%ZjJBH#iI_Je;5y}*Xb@uOHcM%qKSWHnTu}~S zd{nGd)N#7nyz$_0UIS}$K9;ol&N>S&InOoyey+;fQ9~Tka!eF2p#p?dY3)-TQL zykm_APphlYV0~$)TIj)8;-pyQ4afX|dr{YH4D1?{G>aM8`P3#MXbk4X<8|x4o~`=! zCg?@kn&9>ZC?gIC^d6G^2Ijx=Q-78-{oddJPJd=|{u<{0f8u_YH~rpzisN~|{FhVs zZ?#hYUhMSK&OaEizm_}w6#QQK1mN=}`0ok+uLUQ+?WTWyo?k0>{!q*GUX1dW&-14` zo`0A9pNemO%6>0!dGD*y|3&t{mE!#OA^lm5^LzU#7x}%M;4f|dm)!r_Q}3lYe;%A) zi*tTz{C#ko|Bp2O=Te#F+6`u@HB90J<+ zrT3SH{EGwr=WhO$1^!d&@7?V9H>CbE-}+aa@LxOkSAOCj+Ew2V_Wsvr`j2eIUse6e z3;9Ep1nS>V^%pFWU$y>vPWgvc2JFA9^}n53{;KoW{P`a`VF>?so&PnP{;RrQvk-r% zW2N}t)%|5o;&-+G*~$Ej9)52>m&5IQIpkkXoqzq|XDspOCGl%C@l*2eOM;j7?@9iv Z*g{4e^c_P00N~$W>|g)@qf9@4{Xg%J)?)ww literal 0 HcmV?d00001 diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/02_executive_weekly_report.zip b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/dashboards/02_executive_weekly_report.zip new file mode 100644 index 0000000000000000000000000000000000000000..6c36af600d9291b4d0b04b7ec4e187eac653b446 GIT binary patch literal 17404 zcmb`v1yo#H(l*?<2iM^45Zv7@xCWXAIzZ6i9$bPG++BmaySoGp4#9%E<|8*VbKl9D z`RC5R=JaYfy?ULhI%n@)PwlFDT3H?n8WHgG7?vw|^T)~G{*XQYHv!q38QFmBOn_h) zTN^tEAR8+i7b`m(w>m2~E7vPt7Uh3?Im2JM+yvwRG6LCy?f(#j^?V8ZA9gf%Jnl+D z005`YZ`uD&kdlr#P#xrA4pg(YaeiwFvH%0sY#i;3!OX58D@*;TC8aJAw7}zgtd}fd z23Uw#{)%DoaB3ssRJ;KCI+CfWCp)b zp2@;5rM9HxPut3pZc%TZ1Z+`9Nm04DgZFG+2=6a0c>(wftaXvuN8jHqBae%ZnqqhQWyeJrGulRiyeR5s_^4B-Ll zKM%S|8I!-_^BDbnGyj#bvI0AReh&N}1L6d>vp2V~<|BQ@%*xD)=-_G#<|CE(`?N!J zFt-BRJAkZg`ABJg&OIhpcBVgOA0I0h-z#2bUQTXqE)E7(Rz6l%8pJ;hYy538)BhF2 zei~&DcK9_J|MjAf_^;&r+qwALbQd;?+PHuO0J5KzYW?>Cazr%$x7 zf`J;)Vcq4I^pU4(N#Z@IbE29i+nc(+0-Llxw4-uEpuyr+1{m)zW7OG1+KYss5uWVC z4h|*wPdIapL05KOR!wQlwqA+MyECxpWiJ{h^jFWA`eEl#rMEfK0X#<%me?=)^L#?~ z@N0p%y9PwnA{PjVlS`~>b4gIkv!4nUK@efQYX&AXA0dA!!0nyYBD2|Vq zN{9n8VzytfH9iFxlR3-A2HUm-zKo6a?`$Cjh4p$hMz@?~nVqdEW6MPgVOl6=$C-Pp< zRuO%qqk+)sr_5c7A|KFZCkd>sc^pe9W0kRK8@+BB49)DGw`RQZSd9qn7|Z@hWCQUA z2PVs93G!YJa1w?{FfBa*!`LZ>rQ-fbS8_oF%n6&Y@wjwv%Q5YJ_Pu?wve4= z_PBiCRWyyja|0+h6!lU1lWoYTCR7qjT5NUJEo+k~I_@*ouqaV{YAuzBGxbMU$ zBua?4*-te~{5lN_Vb@O-h|++DRt`cU3A&jNus>bGpSW^NWrm6a1povh0RTGx%Y|lV zV`*t_Z3+ZAnf_X8n!`$)tXRuC42)c{lhAn=Nk?KYm2=f%WxiRvN4&EqvFWs7<#p5wm)i35UuZ;sYo}Yy&HiYMXMk2gCOR0;*V7cQn|L)SVvA3dVP3gA>GS!XrxtU(^CL@P~`md>q;2AZL6) zdFQf)p0LKy#Wn%oOp}6~Ai9sQMzrHfYYbxu?a8XxDvnZI&o2t3nGu*w`Bo)AYCz)l zRxMQI3T6~GIpX8EU6j>J_2Sw<2Yk~nE+tJs;@B0VPv2d-{c=Bnmy+r>?>%N^^(^w{ zLl#BOIbf8yYqONa)b?6v=(HNcWZZCjt}(Vxl-)Uy_H+$O{9`;mj-PF>I0(eo2D(!) z20;+>vai|&U9ks(zq7iN@6L)8!p|vxL(B5LU2u7KYbWhnyFVa;qM%x=YP6jDKAn;K zvX|VVnDpIrnRT2Hi&?y7kJU&$JN|KzR{Us2yA9g)clo8X4^U$i!}!1t)dIp#)h?0m zZsX??>5*5H=7C9;(Q7uE<4edk1^XY^oxYa$YE{vBC-?VNW))pChL~tQL^+}W{8GJF z-X-g|E?Zw;f+uYD-WJHCR8Lb28W%l6{qtgUogAOC~Ffox3^WlCCiolBF-Gw zQ~)Mk{qAvIK^4lD>AP(!YY3M*t5^#`VOo?~8OysZ=_D)h2`q)oSw5wjAFxY>V8JFU z+5?V8Eb5hvneirx@5FD1qn{@Hm_I#BJEU`c3N^V3eKeI8WFdMolGb5mv+wcayW7?Y zWwLQ@&4_B@KtF1aP_0a1taQfos31Zt0&ku3v6Qq$B^SKT4{Re3w|ufTkyI{MINviz zW4+CgRy9MWkAV+Rct$!BeoUc3#6`WL9S|5~M{Kr((ONZ$X;wBHZdzLbo6vl$rWjn3 zwZfDBrPi${H-~07BR-KmF(CApWVNl4S6cmJ^}Ushl*6>caWPzu0(@jwGPf0nfptUv z8n$et7n3^X>904 z>qQ8MEIR^rDH&8*vVORNN5}}5AtmXp;BwVxpX7LKt{D1AC;zn1O-i_uan+2E2=B6! z-;la=RS_V^3s!h4I!1Y&@HtNB;#s_tG(GJ}PChpM%2jZcn1Vjip>V@fm9wzb#n+z& z!^l<6g7T?GOK>h|pMpfMFgyDwzxCjyx|u~I3*2l)x5NG&>mk(AjvfH~Ndipc4q;P_ z2)%mkONQw$P$8EK>5km1J0I<9V^e+?H_v{7SqANPcym-u$YG!>ob{d5@J+tnga&!; z=%HJqo!EoXX*|M$opoU* zn2_Bc+T%?4>po4a5K1SR-}`LD)hs4RH{X+HPvr2By>dk602P6f`uu~6SXGC@QNyYQ z67g5zM)qKKGGFo>)?xw*<1~)^+u_y@pf|e!lh8@ot17V2<=A%B{8(Lo7PR%MrJ}r zsJgMVU`wxWQ-RF-mI)hApWozQuZu6s7;30s$->lG-Yv>0M;Pfi+fdxx84!QDiSdG{ z`zV0~QtK^WU#x^Lv(^J+XiAqu^k{B9Wsb{`i$YZ(HlNG~;tXVJPJ|a`Vq_2zAe<@w zA$@8&v^lI1gx6h^9k158g;M4`DMdTX0l4Br%u;gDQFvuY{TLCz+O{T4c5PA<2 z+;L39V=W~XE}l9s3v^DR3KCzJ_ij+9UEa^xV1iB8rq?HLO<13JdUA3$*KZF*M(yZ0>^KERNK zzpUb9SGTEMzT}$8H-Hjnm2l@6wB_X>NYI*M`_Py9f}sMd&J}$S#^v(z{6t|#Ve5$V zvWkluzmO7tQ~703&h#gCR_xRp^|mDKvB`I?BZxds$=+6i9DSZc>xu))*mair*-psH zT01-MkeqC}`c?3P442A2Yx~*(FqC9P-b;KUP2m-xpkF>i(YVp5=8yl>Fd&nvd4ug; zqadnFpW2B+9PrQx4XK_EgM|O)lRBhZ#`|pb!m2gA$k!i*gOi5guaTU;M^GaY%d_#U z0b`x53VNW9jE{wM6hHZOxK05h=I&l`Xe^Q#!K4w5Y5~l9X?8m}7C52x>)S9p^X!_4Tyx6dBtESo zp8&{nL&A6D#8@>aFJitZNg>K6{Eta5x#hl*NeOBPWUuctIA- zAVFq2_nkb8TwiWPtQiu+&g7_>b_abnw$fHNF2*q%kytmGEqXnc{dF5G{@gG_QqWx) zET6KhC>Rk5QP)QzN6cOs4H5xHMdfg=hvaA>b{mCKUAH={`i#hfwm8HWcni7_R!OQZEioatie>)~xbhL;)0XT9(joOcGUUU_>OPF(y3 zR1p`ry7Jf>ymTqVmTu{EV){1=!De$23Ekaww@4dVvU^;aAgCO7C^PLP9s@LtSr+&Z z&KV0RW-iw9jL6WtnhReP;~(y36YlLufie80Yn@N1E&#ss4Hq-jTol^*j;NU%olTi? zewMo6Xkw_BEi$NcA8~wGKH?7Em}}Osy|+-Uu|6`e%bvYiUcP}%AM&Z&l4RpbSk?6S zp*cxj;}mRFNxr3RVY+ac3O}L~L%!$ghZSckucYo(PaB=?nh*w0SXsU}0&5f{y>PZ@ z0WtBpr#B@LQdJa>+939RaZ#*ux_$Db=F4d?_cWrm!P7SR`1Y#!F4qi$eZs{X=kcS% z@5l3kMw4DPO`akO?sIIThs!UZHGU*@y4kdG^Y0%{AX}h(^ZBw`?a^GndVSYQG|uhr zKVP%6(34KwfjYi?`0!iw*p6z9XABJhbi4on1fF#Y|E)KX1vpEEO`L+Qbe>)S~>rS)`5Tk{X~NXdH_z zJmp0w$*1)VW1!&j-Qf@n%L-fPOt$Zq$kJQNxltO;3Z$)F6CK53aK!KjjUi80so-nB zjU|Z3AzVAXWc^CNFoOn9T(Ew)yuayd;GxYTWKjMDQ@9mn9@xjawfe5NHV}FC>Tzj! zd$b^c6I^9BG^CFt9xq%)@AvR3fGbh)OVS6;k=?!usM5|p^OPn7{gsK(gXR)P3(sY_ zpzDyxwQyQwt@(ILj^cHu?=RVUB+~uge@9JrLZDs`#7@4B|C(rrJPTLd&6l+9K%p@> zLW?w)K3oPbFwIoXb0rw2dl6lt{_b@_=Ye}Z{Tdci(p}8(V})4VGJRV>1#5A0Uckki z(Wxf8TiL_r1B(3oE6+(Yis9Fc+r1O2Jb;!Tlc<}oEn=LIwYc=9$#oBQ?>%0QUhCZz zp5Hl;cv@fRczY}ihBdthpdXO{)a+g(sUVuos`%?qsN!crS>f*$K!rtp+_&t16=VI* zwsRCFn_zV@i$p<9t=g~4xyn0hln*mVb0>}4+2I0~S$2sS&VZqlp_{NV?6Y6pL;Nc6 z1|q7MPOYrz)he^_=h=&W>)Sb_?Y4n*NCN7&j0}C|SB(vy!NxTM?jO67O0b6|p|p19 z@l`Yg?ZV)Hyv3@H{!WwV|9YIOWhF1B=uLB^;EOh{H$-OC3{W&-M%m_C%RCAKJ& z6yY^xFpRMM1zyARNHuSF`e$Nxa>^cM$ncB1cqO8x1V{<Rd7&Lb#LO$Y`7B*p&g?`|JZSXc$gsfMJ4R-dK1P-IiM%~)(Fj2XIYRG3mtxqDK z)6datNQIu9GE$}82FVb{?8S6zPo=-R5O%Nn^f{ziZ|Cv1YvPa;q|N7zS{Bk$hESW( zQa^XAj9XV z8P7{=2ah2AN(cE55~v-(_0$`aowNQg{XfgVBU2(eFK}ayEw?K_+d}lfpeCwJz{*5sba;B|naD-C+c{O8tdXMx1$F`|Q(_q9Dk08HLq>}3i$vs7 zonZg~TD1Spl}UiiEnWYJrh%F^mX20npaRI=;W?NF*#UX}=E`(_x-zfpa3mW5fXjZaIIKfSpsv{DxV>uQU4|sb9YWBbgW^64abW&b4by z%;C$jbOIn|k@~%c$NNIuY!L%n3y^Qk>>lHEX@tIdHl^X1UKtYcybN(x5N!ed=! zr{95bEi*}9Kjcgf_9%BY@G{d)+H+^ev1r?t=%Z9Ls(XcW5s$BAs!BD&@!3{1SzT$3 zJMBEy_@dQXRNoF;MqjLqy3L3ldoPt*;W2ot_=qfJ96O`Cw#}LS&PXN#IA_?o)82==$xT6b;bMhVA}zbC*H(^-5 zq0eL+A(oMorD;*rZnIl-*O<7yFMyyLHHp-iEp^I~T`5ygYbztq8MAFAdsy&9n&_!76B*kH z!f^?V)-F`?cok#ck@+?~XNT?Mv`*wmWMQ@HPJ;XE4P%sA*;lkfuc6hC3Dav&l_81}k-uC&DbpU)0eCCpzXU;?HUkP532CXEq`h+} z@P;4&cbU@11=EAawK+pWZ+FTEz#FJno_a{&b6;_rTF+4A`248`lRnE$dgOscO=5={ z3A}?Q^?7|(EpU%Lwpu*`HeP)lF}cQJmkN6k%4jU|E3{n}@2T(?L1zk#c}0Yk0kl)W z^k|rgzEzI(;TA{6{>vTrK)=97YJ4^Uy z&1`euz*Zt*50fg_FOg0xb2X@Sw(s*WN4kt=oIR?wN#6QKU)COeXmYQtIaC}gCcgT% z;{?86euY@&WD3RGT=Pip*4w5z%)w~!Wv<+~xj7#MZmTU=(5$a6ikslCDU55mOAidl zV~pr{p*^`s>AkauZvv4X6}F6mJeH1Z`4#oO8|l3ED63ZO@@Gp}Z2RU^h?2Hc2-JFZ zRZHgPL#OA@B6qKw<#9|XBS){2j|ryKqLCii2XHnHxnGdfis3LfEC-kvJRJ<#Kd{fC zvbaP#zvkcB@tDU)UN@a5N_7po?r!)hy@78vZDBQHeg zG}dK3kDn@;K!o$+K5d1S2Q>&c;lXI$;t}qj-85M; zp_YwPJT$~czWmoRaS&Rp#L_zxl{{L0rKlgC{irG=b=5e#O{MlEs^<~T0xORZ(&FsK_TZpk_$0>lrK`(KiaVsjKQqWoN|rn_kkRwpwrmf%%M3hMaBFXozi$h zvjbl@!kdzzRMZ#6c#s#Ydm~~<6|I+cNi%XGd~QJ|5s~qiPgW(Ks)+??uu$Z(4VH-UFUY#HKuMZ`s`;52beWJRD|jULSR>;HcLi zL>xZ296wN(26gQ43%`fc`O{&KZT)ij-UZn@hvLDO4juwLjmx+r^mrlJIPWk1n!#3{+z% zG^-+qQVQUV=mOJL7|2r`14f`&+hG%{owxzQyQY}%&^j+yzkeDZ!8DO)5q7xqb)e zduI+1FI4qvgbWc*#UadWNCw4qeS^=MPtf09Fz`v0VFpxU$-nVf}C*Fv`yN z5wQE=>@t+`NUw84O!7%_i}(fItjU!bZxN9dDdPZ!QTKZYNwEQ|)6}Y(xvH-ZAyJ#w zU?Y>wL^W;2I?=q!abt%ywje2GvnN_`Dm0*w_0cYLV_fMRbsdxWCubu9op)SrPWNBn z#i_x$e8GiPP&WO_EOprv%kI@IHD1B1Mj1y{Cq|EAY!UMH&gs2GSs3-mf(EXIf(8at zX<=Z6^7}mROF1xWhNAq-+1TtGGOe!qkdej|RB%JH{!P33p=F|CR`uo~K|;Clnq3|v zgBI6TT}t7b8M_cS!D)x!YfaATVH-H9FKjTyc2w1nlEDmfuh3}e@hX5`)RxAw_=qm^ z^u&B{mabxD^h%`VB#1CE1@UjGyK=*%wcATkj=lssMO(&L6c6EXrHHDqHf8z4r`mp# z56y#us6ZZqoir4cZ7}L`6V0=r9XCtVejQ}a3$dMU|COPr$#~g=s4osE8Cbv>mr`5k7fyjk^XLUt0ijhQPf5DzoLOHHBsnLujqhS& z&OSFa=^ASQ-9y{x3lEhJ9#v$D88HF^GRSD6s8Y5PeaOzR?@&GgIXxhE)yWKXo&xBG zw}MKZWS3~5rmS1{>fK8>*Eq~jc27S$E(#I5c>XZ4T|9kh`*yFKH7(~#r8!H9ioRmj z_Uo0BufctTBi~7L>85qRjjMx*9(((yW2@+~V)N^lAO0wV{8_+w1p_ND`3%VlM1Qg2 z=5J+?Upp>UuoKwY5e#%TcQ6BTGf6y`EmZ$j|NlEibN>yCt!xN*(VoW2wQuG15XecR zI=pgF4kgK1q<3ozaCF|zdD}N1j4mRPsXX0>A9}uG%+=C&bl#^TA;23CJ51_l|5QM3 zNZgB#P;`)rj$@hnN`I0zZ!CE*&wi1If^(S{Nn(|E^92|!p_W#?#1&tADrhd_DWf6faa2VyrCZdQXMTCk0*Pztkj9Gu&7|c)*KGxAiI#WNAg4<|r=+wy zz@ipAme%qMq3T2;5H%o88dX5NNqZl?xolTKEemblO(y$va=X|@p~HyL8`fHj#pIr(%UXd2(Z*u(_b0YiyXI;WFz&J>=BM?{`K$ z*Za!fm{Wi{%1N>E0mFE+WODc63`$7wUfO5Yx(#s1o2fBvhh|l@>bcf`J|7!uoaP>qITwy{U6}W=J8xj&y4klpHlwF*7UXENL-sBV|uZ|%j>Kd=p2`3`2ArC(3;O=)56 zNAatX-1t_aJ}?g=uCRCnj2}=b&S1~%$r8d9SDQC?p)i`fRv07&yh@iJAp zz7o@{b4)|BQ1xLdbxMD!e9$MQ%JqJWK6h4SR{X3SQK3}mp{~gZBKtE4ceSLIZMAxf46uYv&@jXs_Iq|Nhg1PWFo(RUO?C!!%%$7WK?a{BBKybJWRb_UnrwHEa4FL zQ+Ch$EMu)Q_6RX^SWx@2KQoen%)xIqt$F(!nU& zTbqDdEM|x|^E_b-92qz(j=%jY(eOO@`u1{;)HDek33PdU>65tbq zujrf~CayO50zbZ>TF%8MzzgesR*sFa{zHXb!J@PDC%&aWP`*o5^>0RwPTsO5l~!VLLnO?0>(eZ{!NfsrbA zDnpOLFM#nmn1cg5k2I!c;%$tw&=-@$AvaT5s_n^vL`ugyu8Ue<8L=Z-9Eoo9?K`UQ z+^ZP@jG;W7$lM!DVhRRgkPfo4ETT`~DCKsudT_3O4imL(D}Jb)7!@r?TrB^xQ4DD{ z!Q_5MDsUlfSq>JVW@%cV8a`c$VBie79zdraT>WOWvz?%(P5f44lp;!lh&}z=XV2u~ zAIQ6xZ#~CtNVuD3jyd@HmtBpe_|BzlRx(^^tfs7Mn>lH70+FnVI3treAL)yF`utv} zctu{_hnLM=5Tuo@TA-D$TG!lFW0rqd%qe)(JbeAnn`y<0)R6V()`KrNf3a!kZ^-?3 zfc^#7e~0OxF&JoNZV9#r^8P|?Z9SU}zMpNf5?l+Mi5UDy!Gv-4TQrDs+6WmL{hf3;h|t-w3U;z2 zcCTy%9W~P~f68d-w9P0O_0$><`WFA*n9~8rYz}x`Vns^5SNoe_Oa8=&QO0t%? zQ7prF*poj+fBDVEI&|cZ)}Bu#LAN*G&N+ZhdSD8coWc+-#wc&xLM8N;yU&>EeM2E1 z!*0u_UoOD;Alc}fNa)&jn*-*lDqo#XID6xC^N z#L8^l>e^_l>WrsE$3hFDyxlvu;*_c~pm9KfYaYA2Nx^~e&%d8i@x3ZpC@1I%3?46v z5=f1vLMcw}fH%C}gsSD8JZLBz3cp$=MLuWzZX+3Gg}bp;Ep< zEIKmx8!Ar<@41HXhy<^Jfrj0fBs2SRSTagCI0T9Vq{6Mt9_se8Qv+%P9FlbLA5lK% zff(5>nKNnCseuZBd=#e)nJCq)ci56|k{O7j6uu14peer=B;O(;O}-aPGl({jM1%Vd zp)5=kr!U8buMD%#>8Ef7n>pz98q>a00X18@x3K6K=1wau0Zy6M$Kzfhs2dHD(d%6Ag^-g*G@4ba8^cnL*O z9e&6iqu(dlcCl<87V8#!dW>N^Z?SrQ%xs4mJEIR(P2kDt67W&C-dy7IOM7umo77B? z8~1$iA=5UY56;on;e$^ zb3xLHIn9ooSNxkxSrl-coUu0P!Za+4Y6n}>38!gG%D}Uv_Q8nO448eG|$IkfqVnMPtgN0sl8umDUzINwS z!ZcB07e(L_lXx@AykO{I7um1rXp^2)RkcE|@IV)(kqZ<}mH!U;TMt$)Tp}GKn+;jG zkDVEStD%VL3-b?XB$NlwrkE7n`^Z<9lkQ$N3fQ+ZxD|3_A0X0GMUGm8a1!ixsH$=! zKin`QC~ppL>cl_n(jPM8MCpT% zz#9sFp%esbtS+4(CXWZvU67M)udYM(nWSctxT!%Jv_S5Xx0_uo=Ev*{dy~D#2m_qy zq6bMh>2602-M<~gv*NPrDr^zP++)Ce((wnam&RWc%1_~>L4tN{pG)(~VA9^QJA{;NpX=*2n1-e|wmc3bf zo9n~q=>jd2C6+q(4(6Vr?h-CEQkU4Lct7=Y=1C|af8Vqn;!?9!qkL88#ct-b@Np?$ zE;rBd=lI9%q4r%=&2BFwzM|XS0_R1+!@7&ST#OVVkHt&ri;ZmdlO}@#;S*`14gFL{ zH{4zLor|6(2U|45R2rmH-6 zVK{+-T41onb6Har{EKk*2PF4u>)3BR*D-gr3ad~lYxC&jZ?;C)WeUlnI4Kuqj^gFx zO(w!R!y=JKW*)BNSbQN?p_d||^kxy-m;zKmkh+)wY z_@oyY7ZT71Lcd44WBNp^E&g$8Y6u|%M*bVdi?MzNa$~s-NG9kp7Xe4OWjlv;lvmlnK-+*evEQV?=I9&by?}yYNSBV;xp_xkxVtVj%8)q)qxnfr0QX zD2chzgpM*@rl6Xl$pQ;Rsu()6JvpS}5yOeCVpJ4KPQkQy`95Z8Lg()DFP)z5Fs_7G zS~C<4Oi|kd1U;ezJ)qrT_~)WCjd-E-(P^SjroOAsB$B6tK@pt0N$C(8C=k zZ$bI^{&bG{GshtMv}u;>>2+n!F0CZC6_MEY!8Duj$xT&jCY;}WSk#!s6~1+N>yt}@ z&`d_+bhIc&IRr!5zWNfF(yQwwqY9Ew4_`1v?k5`h*OG`N+t~y!1bItkFaxPZJ)q3L zkxh-{$Sw(H6o_NcR3%35e(uA{hn0SlE~=mKMZR_`Bd?GkSHkLczQ)Aswmp&V07XG3 zD6uNSuy`n&yr38{K254BH>NHC-sMe!!sd*$kZM-Jdz50nGw|M=!>)Vr4!isp5Anyn zSjWS`?b~^$rX#EIB;-DdYtQ306ymBOwL3Sl4T!mWAm_cjSzib9Jd_G{FuO5t%lC!l z^y+}P#)XgRR-mrA!Qr%#oZ~J(tk~frR99TuLSflG5LSC)qYHqFN#-cE3m&NDUd zGCz|b47ot89b`tSir1!{sDX6Kv&9_McN20{ud=!v)o-q|hh*qlnpx8QU7V9>kw00G z11G|Vq#tL_I8jvN^=*frj8W$i&0wk^8(`{}Cu8ch@1o`oyj|XGAcI}$cnJcwBq@_>pvArN%uTF80B$H%SIp5B0d_;cl_(p!(C-I`9=rNexk8 zMtxvFQcXE>bZLe&Qa8i>v6`s>;R`9;vPs9oC8T=(M1@0@f&?gK&RSCk#HGQ}FMBii zZ<>j87V;aLR=Mwr?{?`^go$1sW-TR99V;-Wr(9w!WLUUsIk?rgPAXR7?JvFUcjGu% z^jr5F6|3eS7-6#0T}~Xk^=)Q+gQjvB)tvmm5I6#PBrTBMeD*_aD8XSGc42IAisiV* zuHfwLD{H^<)!dpos{mSbBy$Z{^iavBttHvs>;+fb#|@DN57`z{`C>}BYCWV)Az8oN zDS35ZJLk(P^zd^Q&fT;#CG&@8)+PibG1R~I`#-t9zYa9Oq3F-Ef4Js<&;0%O;(l^} ze;q%?2|u6yM{)m`3H(#=pSjxKxxhaKKhuH%-af(qo#21V#{2_fet$f_(^~&j%kj+A z{EzqZPbAm>EBjwDQ-8_^0DeBJ(Z3@5|L|1*{TTdYs{T5D%Ef->+5AVF{~`C^KB;Hk z>OT+8@66So8h;Ir^Z!WWzu>X{Zp81bzCVpvd(IU6FK+t}BYx-j{cgtZe5OCmuzFr~ z|1&fGWH4wu_jZv#^=gp(b-n-pnn!-O z<@Y+-pSJ8$|8-mb%c|M$M*N=g`_qW*=ieLns}tbANC5sa;9n!<&mGdQyrwb**%4^|cI~t*vbBwdiQ* zm}uzfnB{4hX&4zY_oa(x|p9bFwe1H0c0A-!J${u_?wj+X-gKmdTt_m9Xw zGbE|PrzNlBY@#J+Y2{>Su487PC1>Set7kyvqGMsM5w$GY^#wlQ{22*@I!qG@63I^@ zEM7F$kB?U@SEf3tPR?ZIY|Ao%FYMeBj~B>}@p?4l(TLa|L3d&R1W9;smheghd^xox zC4bsl1b2sc`@(;RI7*Pn+0Ee43YJ4fw^C1VH})VWVD6Zpe0;A{TM)It8B=I_v6?f2 zV?AT+vrg-kc2@nTsshi|6z3cg%%{$;(U|9O2Jlsj_6!Yc-d%YNPdMiaC%3(#nOBAe znwbnlL|xXFn~p@^32ZxycUgL3V}~+9Q-@*X1ZgJNzez^}&I_f+W>OD4MdyNYgQ<*t zOF~YF!p3NzM3xSX0umGc?p1|MLQ*Wk-Ht49nFPgQ^m(_JGBAIw9fg7YlkN~*(U`?L z7lqyx*&$01EW@NFqbt_3!)h5;!D{@fnG-_6yyNCra492Mni!8r&fW~>G2CAl-N;uD zSmJ$+etxL_&RSU**z5dU_}>=9(ZJTu#LAKlmzIi#iU!i&#oB-km;Ybe4$|Jl!obd6 z$HJNo_w&!aM@d6Z`P=SeqhVxYWTj$ZrlqH4B&VTaqoMf>`6sZ(Up6z@-vRa$l%0Y7 z@0;=8I|ctgvhyGJ;xF6XtRhD%9|!=z1{?rD{=YPXzKxESy{(R+p^4sa4|-Q#!YYRb zzV({CBzB4>MZv1i9U;ilOdj&4g}5(!DoWg;tsQEF7(O>fY#o?&9qs3py{1^i`|DH_&*R|^ zAHLhCgiAj?^zj-`gPvK9naIV|X{ggol0=~fVpe(dSmo?hF;$X_B+2vZXKt(S-^D$j z#2naI7J>KY8e?>Y=N2UF#z_d7aAD_He;>3qM$Qb38EiDjd6{ac zr}v(-gmO?FTAOaQv+qb%Y(6?*Xfb7zb&Tg?dtN(2$JnD}cL1o$FXPc~2DUB%;ib++ z5$Dj&(*eHAK3KaKo6f6)X}jC*fkDM?y9K896zV}TxZ* z#|)uyIIq3K$LSn2gBbJJOThVh7VX)o!A&+J|2R;~y=nDXL&$QgocG1E+u;gMvx4<4 z&e)9iS@x^g{#4~sglsm%nb*1=SO=bxHXK-WB`11#zN2UXpWB^YDr+?k0iBKP@h^Tc z+OEtR^X?Zy|7E{$G%z;NGyk0te(?*Q>)D;Y{M_)^Xh>nDIDC~p=x`!E5+;-CcF$9Q z#Tz}TR(Udn=ogOkr2Dh-Zlu*R-P^c=zJXV;Y2u*+`%CG{fX`(Q@xyu}fC!fEyUld%b;=a^kY1fi( z8Uafre8=T_u?bl7)bxuz6TVl5nguDRe919(a4UusU?FZ~?XDFcZ^CeE@uduE`o6KM z^~RiTj$QJd&LthNKXwuP52m2;iM9BzX-cAj1P z;ISR(8FQ7$kSs)^+()8JiDG7hMWy6)Fb`}yuZyPnn5EL5?Opc6*i|a_B{Nv88Typ>5Y94$WAh zT~>^dpI8OB7cqPAGCmlvL%+kQeW{ILV2(aRRdr~k*}YJi+2Xcn8*S^`CsmE`{l1xom+HoohXb?Ad_Zm_dZh;*rCrWz5@sm24%7M!PU0}FG%szOZ zk3hA9&O4*IL18}KdsaN5_?^F`Jlbz=KWv^|yA!-%ewf!vsc@;he7$^NSarFW4fy`Z zP_$yVzfvhK1Tpy|8P)9ofMo~hB%K|5>BaMHN^)vAof8|nh#vc9a6qJ_{U?q6`7R{9 z9tbh>?gZ@bPN4i>BnMj^dlO3|EqxO^`#%U`(sGFg$zxne*@ILZ#Gi>~NDVS86cPeb zXO@s~ctH|_(q?KQ%Rq}eg(V0GF6K$rBp!cp{_c`cfeJcJxFjm=gS4K_G(cJXXi)u9i{)Zh4@|660iHk2 zm%BQ5WMVH%nq>RI471H}nkMnOo<}{A@vSP){GVSjE`x>|DamtF|`bb%90aZ@UHoQ#)`9>E(39oqDmhrkOV|rz|s( zn5sF8WySn$0s(~BsvgTVPYp&(FL-p{I8Tz|EjC%s)}dYRCI$)_p!Nv5L2l*i*Vi~@ zA&bU|_a)?*xZ*ILY8H@%3J#ldyC2(dBFED2i^5j>Lef}3tQF#8H&^lAX$HWmP`}v9 z5BgY}AY`O9{fHtDadQS@wfcrpzKgrC77mvp7uDVepwQ%WS+xv)5kG*`2LvLQ)U0Hm zt~_M)%({{)xcCmwopBIiyt zXQh>q(OxSv6c{t_5N;pDC5})2#|zv+U7~;}nT+d1R}luhh}xB@q_a;Uk&Oxdq;QB# zcqKvjJ|NQksrc8^g!(N3F}MRV<_x{S=YXVedg38_45-r3qkA&34OQWBv<`P+L2&_V z zenE#q(Lwi5tTiCSgsb6LsBWxPL0p|tx*xMOT_3qE;>#nvOCKcTZbOT|EJm)X32w9r zYE&!LM_Wy8wZ_-tLL3y$7*(C;N-bmOKQ+OkTUCkI2^kr#)nR{fT{f-%4g_UZy&PSB zzf;H8m3w(SbQp^%KX0N%+ZCn;?ul}eW@}rcx|gpt^0vNlH(=qO?q>n~-ZvTNQ4tEEF4y6is7pSaL5NXowR>kS@ z%608dYajAjwQMTF77iOO(piv9l`b$97SP)ZWG{;v2Y9CB*gTMeRQBG!O|20hhMO;; ziQ&gvwutK%90d6~hDXj=-Y2Vr5LAKFwAu&mRa3UtQKz=0gE6qr&8t(VqE34lW-pd) zb%{uJmz59+-Rf<5PSYC6IHT&F1)>gaIjhDYCj+3J<*z;_LFo1}r>hZxrtRxEMP=Ma zMcbVd(|upLR!qvSF#b%Le$D?${*H&}8SB{okud$qSP)qs87M&k0OC;q0C@g^6*&i6 zBLgj67cJ>O97>Cpw(;otJ}=uDfi{_EsB5Dt$__<5!xsXK6Orz; zM_hnz&;h@SW&m=G{bLMU5)noIbU_^{P<2?JIA50nYrVLq%u8&J9gN%<8>_Ie!UBZv zCk7Ik0q}jtr`V=PT;=tTAaI~DQVhtnOWIrs=CDf(`?5%^f<8EIoz8SwF)^33k=LPe z3BZ8sYB+EY(3eXL~RGJaM$JwkE4+KIe)L z6Uv@}+$DyMl2EW{W+OCV>X{MxLez52P{MpFrX8zxkRDC#(q6>7Xi`G~n2Wb?r zimF8lsj@-1NtdgEtNjmN!KIg4LN`}WTL%|fK69CG5fe`~wp^Y{);eWtAJq?D>=eS4 zDv>*_E2ARs7dfS2zmfts{;2Jl?I15(9*SvY3cpc}8%O()W^!ag)6mFBwe25NNN#h; zuq90rwzr=Vhzd(roVZpg9LQJ{pG81X6J^ZJk(R0zCUwyJa56x^$#|t*FwPH=Es0@h zHqJMYD8gUWm=h@#g1jq|Y@9>RqA0HGZvty4$U3~|DkIr2`e6wfzjQ~M09%wVS*cJK z9MHnY5R>Y{FG$C*z#HwtjTXeGUq8-=(-dOP-=kFL;)kw-6lFe;zWSbJ{q_wdLQ)@= z&c$KUOcdTGy@)Qt*-dj#W`bW&pRi^V2xkf?3Y~^-3!(V8i@l`S-pUMGiWT`dAN$~e z@T@qm8mi6`c0{Z<_rYO{gavjI#NOoWE!9~0m3nU?UG%^Qms;&9KjZAL9Qy%=Cv8J8 zlCjo38^VZTAX4N$hTE3&m}yPfi02(WHG=_!4Ck=(<^)}}BEvp$rTN?$RL>+AEgv2q zXC189WaY3SBwm_x_1U=@6jm@c-#+v!^pYQzqFd3Wa2J4dR)&`h8WHjbF%*rQD;v=o zF(Y4L8b-T$guJRE?cG~lC?Ol9w2;v|5me=eqPV0tT0h-a1;IZ>(n`~ArtoS{c#(rE z>$u6mZ-lR7f>^w`f&wq^4kJVnnZ$IZz)3!A_?0<0g*JFoATB9HcgG;IGbnAH6UwPb z+{AlAeE$B}_Q8K=#vw0{WAUIqiHyi;PMt);jubxzJ;ot&9N~q zogL80f(reZjp0i*X<~Ws6}d+lMA0j;HsZxqRuc~O37-SEe0^S1c%98x$EH&!RaEHv zQirgJ7&VxjV-fTApl=HSlPhbrp%~#I$0+r+ZP-kPpxe#H`}$ElceISWV*S{^~YE^npGQ3Tz8Uqwq$m}^JqYWHF|H_ zmj#j_lU=LgNUzLs*)-g|3rUk_7_fNi3O?6=+#9L}~KAVHUDu4je*xl6czWt=l$o z(ICwLxnl6)ylKuV9OYATYrYuDkD|K0kYq*L@>`4_zlMOz4miULkjn61sS;K^KOOLQwy|e_JV?+R^zY&)w#ns} zaIvCdDHVL;Ufw^>lZHATJYp(Lyz};2wz| z%@s}O<9>AA9Fshwa$6p~tnyK@hcqlBf5<6R&HeIm)9|n+Ft7HC=^}zfuQ8i8C|GgdXGmJM;p_CFrj8PR}@)8r&dYLLo%CWtbruxh+k*A`XX>*+?h{| z1Y*YKBzL02j4I_7b|fQ+BulpecPk^7+y66RN_ZxQHB_E!0~7v1(y-p3F~+S`vzD_T z6lUhiJMwd(CIk9xxA z{x~+xTs9g;H2+SJ78V){Sps!(jQ@Z*rqYR<7DMZPa|-W}bikn&=~=YWrK&1#8fgJ0 zQZDKYcpoQH7**OsFla{R@X25>?sPKsWLY{hfB{c-mDk^ytoHs(6A7N_0XL=7h5)EXM*X}LZQk?IZh()_mT3 zDu#L3JLl`LC5+H5qaInL447|J_#xjq(QfGYD=OzwPGG;Sfry4|m5&dsQa!Yx<0Jad zJl|}9-T}Gu@YN7SP5@niuV>WB8&fX<78R>Un&|D<1Nt``_9ZBKPPsjumN=T^yjXjB z1&_B=obgO1No5{);2ey0u;QQdB2qNG9YeNuOk#Q`vB09D0bE}mfMD{XhB)}>jMB~= zk-})ZsDvtPd{2wqpav*5WQ2*ZAfn{RQm&tdZ=_x1ILgkD-1PDmYFmH}`EiX0$>^y* zO(Cw2_mt_(!H;*rfZ{PyX}bywvvsh;rH+F#vxJiw9$5v&sJRl`3dT_31x26v$~czj zNyCyY_i$T_r-`9!T&KpPh3Vaz7WRq-+`ESbyj5SZFjgOz(CDu>Z?I>PM=C*SYwqHc z61gYc&yNjVG9#{-Zs1)phSAZ`6}J&QmL-X7PX`7~T<0d;i`{sYM5C7$N4nbCwYH+} zZ|90EKm6&|4ujdUfu@Lv$khixu9IAy22m^pB}Q4=iEr388ws>It}S#^gxxs+y$qzl zxB)>pGKNT55AO|};s^AYt9=pa6dn-})bxJ8{poiZy*Egv@@gUwj0|48MqWidfk3@2 z>$MvaiRz1}Nrq(XEFWRY93E~gTTb2_jry*D@h={ddu;k|C}mEL?)j}a0VW42#^4fC zXAKYx2rLUuFbr8f2^xGR(n`mp#MSB9It<$7q~Vr6*m;03*2(Gk4Gm>`czLwMcm+IE ztHhCOxp@p}$;y&bOfjDsT-Cabr3qeE28pE6%NrfR>ag4gjBhm41#1?shN7l3>jI$* zOtMd<(Tzd2#oi_>P_2^;UjX9{!6vuqP})?uj(U}P=+-OjrA#HRpNPItL(wR-N%KYwbRi6##2h?AA zC<3-2Gvu9z>ahR-^!@=4@x1en&U;qy8z1pI{482oS!(H6>TB`87p`AL{h7tI0)dIi-KU)!zUuN~Lh{qXd( zhcKR>kx!pVj@N?VC&2QE4Zr*}(F$ZMoTpu;S!yw1p?@!vq>{(8wX& zJuo*OFVGSm5*ip6m_+TW7Vt9FXNE9qYyjOVnVuwdemUD8c)7Lr3cJ3x;-N0SpR}R_ z1l_vA8-7y#g*P70vyMnWl>SQefP3 zQ;?x#5(1?qDSUDhH>`xCj@SmhBh!?i-wU<6?oO=!4`Z*S8*p!e8(QOM{h>Jn=RXRx zwZw+2GjaTyJw?^Kn4Fu#>j@40h)a4naL>VypiG0^1WfU@Ceu4T1zLPOdr2>(vV+9F zPI;dUyVAb^yN1X|6*;xCDG}-Q;)MpVr+5e5Fu)m3Opv$@cWm4Aq^n90L0S2zN@mgE3|(vt zcyLQM{KNxLb-q9L>ck8CxGuxQ0MOAH-9^Q2yg1W&fZtYXuYNlawC&lMJ&B6}X2Y+E z<0@$34MGliGhe1D#@CqE!*ywHJEm=y&@2$~8~c)#hg8vmUIji6aK}y1(2&3#fi89x zwPaKmL)Eu7l&k_okN{*Y)H4WKLEO`U*Q=naE#LiJh$&6NA<1n?kh%?7Kgo$?^9^fn zCFixWBI)mipu(776wTI~UC=ZmlNNKbBlkmwI%C$TeanN+T)o@l%y!x{W276iXXYLA zVB;TMTUC5Ie$=D6`j#3%rt(>VTG>XjaTs9s`!B zn`ky-mx6Ri55@bNgYF~7rX@ap7k$$P@FM3d7IvFU54rCii!GO1wXDV3tudt^G8yI4 zKY&DcLG>Nw%V7~Oo8&(D;-7JQCl_T0&f&K28#lBE>`SfH$RG+E;VFkIV8>HuNm&7^ z!PCHhUzKU#iyIpPkRladf^tI@@hzl_Szbe6<$lwA+??-o6bs6V0*_oJ;3qqlmta8X zki~#u-R>s8B(`jx3K@c7Qwb{a!%5L`u<0?>9p3(E{sRd0TaZYbff^+P79?~Zwfjl9 zpjd_}pE*wT&AVYW&bHa@OIMD`fCMrBoO-5Pt5|_kwl}*_-G`VEphKz8GbN#8W|SdM zs8jSrHa~z5OZjHmCXR zyb0caT0S;Kw5l&qa&}Wy>?)F)Jfawl8sbsKzU*e2`joZn8~%v)px0+~un4P=7D<|$ za;Y**2h1RkI$)?faP}4Ixrw+~4s!CWqI00YC^M(eC?7ZE5>5K}nm+cvza$J3-+ttl z1sb+$8hb{A7Mky_kc``eOdOVhST&E16F8xhpOs@+Pi<^0HqzBOz}D2jIe(9xqwE6} z#ka^Ikx#<%{YH;uVAIw4vKH>~X0riWQOealfyzTQWJE{wiutkvhHR?bw{V zdYb-t%F~ZlJHA@2xs1W`T6nI0Gxqc{hx1n|G|_norg*19mG_kFXVb+0*<|xak?mhp zD5zum8|nR$lrj9PRQ59|Yx`GH#vY6z#9S{Cq8(G0($M@zQbryODMIA<^~XIH*KNgg zZ+bl!;T-gSr^`|GhzhTb2qe1DH5l1IvS0%LgmG=?8pMa#gLo3fGBDD(VC^{Fmav?v zv1=Y)e_ga3Qq9w*T)zW7mb{P(b+$F&XP?PY)Wqwc&DX|Im2lR!nKRa3Z7KnpxPs?>D-edCQ+p;0 z2Ai!h#i)>o>4FJ}OqX)hfURM`r+MPza}6MMAg@3be2Mnxa)K<$dGh=cDZ=iZl$bNh zsJ)Q|AjXfg!$$8(@DG7W5rYb0y_<_6(hc4yTG9|tQ&|Ekx9MNz-P~E*TdiAOW>j6b z9RpJ<)p_#1h;~`r8c{lHn0%*17u!D2Kf)n|J?oVbv=zaSES8@nQe>~4tS1cF3Jdiu zj6h{#Ntnmk4@ChjH-RAa;fTOO7Ip5;E5D5Q6~Y%C_(T=-kbc#Fhc)O*d4LYxKRNm% z6pv?A9TQSWuE0O#vF$ zHnirKK-oQ~LQ%xPP(qjbn#tZO$~l0S_*pnG$2ljRSNL}9;Q6)SL&&jvZ0m8-WI`7; z=y88JsD<;C%t}`q)y*h1JW#O=nLyz79i94sEq*~pW4Yp?F&F@kBB3TiQzl+;2&6Ok zL4CJTSHXIYllsfiRa8`Hj*_zzeY?g6T!z(}`br(-$?!^;VZroZiL*w&r8FB(H)@lD z@265j{JzQQ>P1-F>MeffqL@w(34bct!feV!uUo(5HvUUQUDA;80O5W3-hmRt8xHBDNF4t#6PbZ&!t+F%k$t2#6V+z)dluBFgHH|OU8)xHYH_%FU zW7Lp0K`dUBcJShbA0ap@_}zZrno}`%SiI zNRzkM%lT%k`XdZ*ou)YeEu|O$jbvCN%0U~d9e{hjD8Rg3n|?yq5F-g`^{_P3bq@bh z=@JgGf0}Miyi7N-(pgDPd(FE=FBgirjOxXVHX085tqZ+Eo?gx+zH`rEPTPxI9U}bz z;E06>&2C||2L-Np0aytS-Y~P8E>DeRAtiD6&QG&W*1)gHC%p9|mAJY{aC&E}{Ckuh zah?L{^+R6J=&wp85#X^bTCVUIAH(LuhU~=RKms~~QU-@z)bP^AX{qUVH6GYE#<}$N zE4t)s`2gecsHL_yuMVL`%s;`Ph*6_F^%}I0!N}RO1$PFFGJd#;Y&`>nay+X{UMS|&0DSYM@uUd|wya~g;<$^+UNUy8ya8eo-{DOg<}I$GExZfwFE zC;6}P8#jYhkt~@6$Lfq+?q8qyeNGS(eLTa7Ub{DyzWRMBmKbq9^zJ6k{Fa~D z{VWyoSX=*jej~19XRk$1qs6b|Vy8vR{4a*Gdr!;Sj+LyfMcDORa+Mt^)?#}^IJW~$ zhhs)^v7v=|5fG!b0VnE?F1^mF0DP&5EXKoO>l#A)x!P8DKgZ?K7Jv@Be2TiK7Ey*DoyAlW1fm}Ns2_=KboAFfN~&?M>d zp1CU_(W?t0mTjW!+QS`c$)Afhr1q;3*JiVgHg3`Fh6svBh!U%mb;-aJiqjL3p>=N^ z`pJ%FhkgBl6AAA7c_>}B!Pdd9E%zMtJAKq9e$zJ!%EBjlJ)MfwwtKN61X;QKEt-j{ zEzU+x3=Z$YQ-w`QO+u5(%V)bX19S|bK0q%rA|=SKO1gmcbS*0Ci9*~U0mWSb<2Eor zF>K|Jd^D3T`+31B3U4G;X~?JcEdGuCL~A{}tGBDOLneVPwO4%?5zc9b$zmgc_o&<$ zq{Rs57K2lS(1zDfJTg|=pGs8nopVHL#+tiq3+fL(Pp9G&C!HzlblaV!br>~mg64u; zvDi$3Q4wUB4%F9i)2V#vGalQi=jW*7)K3ZD!Dv8{b#OA__fD$z=ropo8cd&e)ES;> z7B%^%RS{%lAz=t={X{K6=gCN>Gi7ZsQcIG3C{EgzBGeotI88D&%dEL^>(Bu+fHF*^ zp-G{LQ)V@xA8#%oZjp}(y&a|iDfx1AAl6eAa*UUXVVQvF0UvM)>x0^Mbbl@oiZgEVg#}3VNv5URqO@PfOr3NI*pM7x`f71e>wE^ut zzF9=evFp!C%ui@i*ji!&$To{61YjXMAr;9}{p^%1TR4;M+BuJ&X6}G^98zq%twT(x za2dZZ_#sI~y`7J=i8RDqh|xLOp*zlo#II-gBxe*-zKLd|w(GbqR5jM({3`zp=^Y%qm8rKWcTvAi$TZ8cNsCbSQryqXEvF@8az2}OZls+5dAch@V->#LZacaol!N&|6&f3kHFXdcK1KZ6b%*zi^vd9eV zCQAR>Qj+9edzru7%8Arbw@#f*HOjUcZBrYC~Vnap+2OUGHDM>rCc zU!CaTXuP#NyEflo1A*-pI5Tx=ssM3t@2nsfGB$zdmT4Dj8hDy*$XDuu`BGhgi>BoV z!&Iv_(erJrGX};{$Qer&OXg_;()?*g7%B6DXwyG2eW~^Euac+{$ML(9q4nURM0oX! zXCA$2Tde`Ze#A^ZeF+M@_2*6YuJE(%5>8kN&iK%_#nmUaq{UC}Pm%UiG77Yr;7 z%@BU;p5`>`!`HW?q(JG6?cGpB%73&#d|@%!42i^!3osife+L%Z4F@Mu`oEt zttw}3?A6*g^bOM=1H!+ky+BvR^1^@mlA_S49uM>_Z#6e&`gE_}k) zIrvnRH;;wm6Ir`DoKl4k1A)JWRc=mCH?=cKQ?(QVi}zJOv8t1s#ZFDUM3Wm=*Mq=S zd_;0MAgB{r=-z`1^rD1r_0ubICSS{fzbA|!UeNI{L}L9Po!g` zd8Bv(D0Z>q2BFrCKZuYl*J->q7=$Zw=EOmbAHCB8kC%w0Lm>h$2j5;bcRiO_f_8Fc zbG3XpxCkQk){yM2ytv_yOVRtNA?|)|dI1%LNCpaZYseW{M$dDN>6>PH&RnpsWMo>o z@6SOMJu~0V8Ocu`9Z*pXSphAOF;u9=@Q|2~G4S(YuH1p8*h~|c$2q9mVEub zDoAY_M?x67NIQnt*~fT7Zc^L97h1niKfHkk;(93E?8;t-ctTLXDdt!BKH_W<~ zgBO#lWNCp;LMOwP{UVS1Dngw`gM8-2F$r{3_+%Ai6{VpY_bi4GU+rO^E~K)QkxUA& z%pGg;@uV`%lwl2M4mH0uOq@ysVS=rb_D~n}8$GyQ+1T2!KQfWvyo8c?c{#fL$k|>g z9SD_PKmBY@s|=gd3{(zT_h_5I!BJ9;egx29)M1&&xDrk&#p*?TqEEa6pONJ(7{hvKP z4J0J4Yc;1 z?2uNR)bW^c0rAr1&&=WByhJ`W~ha6y!N4Fwm_!n4w(D|)qBDTpLFtfXP1fpe#CVg|DzOrdCvKzzMAWR3Ov%exH6^$+i-G=$XT zWyNbv**(p6MUzy? zgdOTH9x;WpZeWn`ZHS!+yV4gByGpQs z^{j7DbPhnw=PR7LiNSka&R1%vfJKEnRI|S~bsyzwTpN zM|nMac|CUQI52bYehqt~bh+U|JsXQyc$v^lZ);%3kyiy&afSI7t*_A_1QKvB-TOO_1GxaKrx zK1C^fN8hFd2tGr-T^~6z2S0;{s*RuPE}KtnP_%F=O-gKRnIWLHW>2jZQk&Qlpg0C^ z|3ZyRPL-ky>r*%$F6CPUHG|RL~b}){tyX z8qCvxoJ|>;6&SV)1p4jgPD!tV^~rM_vdT-EdNSdjYHACZzB4MlKfE+yJ7_N+w`38NS#m(t$ZlVUKl zw6ciWWKpvHd8i66k&t1eP6Rvg;C*;`a&aL@r&4zz88MJJf2_v-r2iwrV1xO>Em%*p z#3yCLS;IQoiDX!$PYF~i23vTDo`6F_i)em)rKPsA4u0d-GnufMWQAL}7m>JBC~#a21&ul|0r2UN(T6 zagFi7lVdVT`VR_N#c{|0RU5RUiSW<+>r>JKUI!Ef@w`f{#j1hTx)kmTt(6p?GN?na zzliD$=t(YsP=nHF;R3$Yh(p4cF+Xs8v!Js|J+7f(gj6`u7KXv#;j^(1IwkS_mUj?a zoL{W7MDfsNQh5L<2dkuTdT}>Q#H>=Y zlK42VFicMx4NL}Y{L3yRgY#I2aIXC)6v*H*8i|=N2Za;h-Qa8=`zBV{9s9@P=^sWT;uV*5aVg0IYQ zlK~Kwa7^CTRo}OjfdgC<322N1V$2P^Q8KbuHNrHb>Wv?R3mmK|h`9_;T|^qeJj^T2 z11(g1c+kV-#hU5E&d#0Iw1(S&A( zboR(Q6&F>hU6b?OMlJgC)P2cDZgQ!YB98D@j{Ef-rQzzO#4T65#Ur3-;*S~3{YF^D z-;Y+q+1lJ#iB*#K39wNqhM8fraiVJ=xg)3H`_;aOrANjjrLHD#zP!l5PMmeDU@12P zH-9v&@kirUNPOk?%c`TfIrc;AhoE2X8bL(}@FW>Ag{s!%OMFbEX{H?UbkIRgcKy0B z=)|rn}N zC&5=Hd~DQEx_D}!%g0822$lOU4)aI_!4FIX&O=Z}YryMmP-ak;7Gw{y&KB7hyH(-~ zjxwz#3e7z7s8$^an0>R;)Kyh%ktCtkXs9VJSx3wFmO?9qwvW$|Pns^}_OI`$3?L9T z@V^h)c<+Js_W=)Z`m+i9|2}5=f0_HK=Kg*BG{^n^^?$zit0Jt?^IW002Lq)#%@` z{lApQ|9MG&YL9;(KkXs|{|(IlWA{Hksdv`@>*D;XMgBSC?~CK~Uz+h(W%92GepP+` zi6HVlQ~y8L>2KWgPXzy41Ntk1UnO^cB8Y|f?;-eAfcGnkU*$@FqF96e@1gjMcVH&={5svQEzo~XH}!th@qg~8 zKW^f$Ri0nY-hY1?{dMABd*1$>SQYc{PyDZA|NkEyaK8fjwe#RlKn4W=9MJ!>58>CT ze?49LbLt + bash -c " + superset db upgrade && + superset fab create-admin --username admin --firstname Admin --lastname User --email admin@example.com --password admin && + superset init && + gunicorn --bind 0.0.0.0:8088 --workers 4 --timeout 300 --limit-request-line 0 'superset.app:create_app()' + " + + superset_init: + build: + context: . + dockerfile: Dockerfile + container_name: nyc_taxi_superset_init + depends_on: + superset: + condition: service_healthy + environment: + - SUPERSET_URL=http://superset:8088 + - SUPERSET_ADMIN_USER=admin + - SUPERSET_ADMIN_PASSWORD=admin + - SNOWFLAKE_ORG=${SNOWFLAKE_ORG} + - SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ACCOUNT} + - SNOWFLAKE_USER=${SNOWFLAKE_USER} + - SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD} + - CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-} + - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} + - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-} + - CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-8443} + volumes: + - ./init_superset.sh:/app/init_superset.sh:ro + - ./dashboards:/app/dashboards:ro + entrypoint: ["bash", "/app/init_superset.sh"] + restart: on-failure + + superset_db: + image: postgres:15-alpine + container_name: nyc_taxi_superset_db + restart: unless-stopped + environment: + - POSTGRES_DB=superset + - POSTGRES_USER=superset + - POSTGRES_PASSWORD=${SUPERSET_DB_PASSWORD:-superset} + volumes: + - superset_db_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U superset"] + interval: 5s + timeout: 3s + retries: 10 + + superset_cache: + image: redis:7-alpine + container_name: nyc_taxi_superset_cache + restart: unless-stopped + + # ── Trip Producer ──────────────────────────────────────────────────────────── + # Continuously inserts ~60 realistic fake trips/minute into TRIPS_RAW. + # Keeps the CDC stream live and the Operations dashboard non-static. + # Scale up with TRIPS_PER_MINUTE for load testing. + trip_producer: + build: + context: ../producer + dockerfile: Dockerfile + container_name: nyc_taxi_producer + restart: unless-stopped + environment: + # Snowflake account in ORG-ACCOUNT format (e.g. MYORG-MYACCOUNT) + - SNOWFLAKE_ACCOUNT=${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT} + - SNOWFLAKE_USER=${SNOWFLAKE_USER} + - SNOWFLAKE_PASSWORD=${SNOWFLAKE_PASSWORD:-} + # Uncomment for key-pair auth (mount the key file as a volume below): + # - SNOWFLAKE_PRIVATE_KEY_PATH=/run/secrets/snowflake_key.p8 + - SNOWFLAKE_ROLE=${PRODUCER_ROLE:-LOADER_ROLE} + - SNOWFLAKE_WAREHOUSE=${PRODUCER_WAREHOUSE:-TRANSFORM_WH} + - TRIPS_PER_MINUTE=${TRIPS_PER_MINUTE:-60} + - BATCH_INTERVAL_SECONDS=${BATCH_INTERVAL_SECONDS:-10} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + # volumes: + # - ~/.ssh/snowflake_rsa_key.p8:/run/secrets/snowflake_key.p8:ro + +volumes: + superset_home: + superset_db_data: diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/init_superset.sh b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/init_superset.sh new file mode 100755 index 0000000..bad988b --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/init_superset.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# ============================================================ +# init_superset.sh +# Registers Snowflake and ClickHouse database connections +# and imports pre-built dashboard definitions. +# Run once after docker-compose up. +# ============================================================ +# -u: error on unset variables -o pipefail: catch pipe failures +# -e intentionally omitted: non-critical step failures print a warning and continue +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUPERSET_URL="${SUPERSET_URL:-http://localhost:8088}" +ADMIN_USER="${SUPERSET_ADMIN_USER:-admin}" +ADMIN_PASS="${SUPERSET_ADMIN_PASSWORD:-admin}" + +# ── helpers ────────────────────────────────────────────────────────────────── +_update_db() { + local name="$1" uri="$2" + echo ">>> Updating connection URI: ${name}" + local db_id + db_id=$(curl -s \ + -H "${AUTH_HEADER}" \ + "${SUPERSET_URL}/api/v1/database/?q=(page_size:100)" \ + | python3 -c " +import sys, json +d = json.load(sys.stdin) +match = next((r['id'] for r in d.get('result', []) if r['database_name'] == sys.argv[1]), '') +print(match) +" "${name}" 2>/dev/null || echo "") + + if [ -z "${db_id}" ]; then + echo " Connection not found — skipping update." + return + fi + + local response http_code + response=$(curl -s -w "\n%{http_code}" \ + -X PUT "${SUPERSET_URL}/api/v1/database/${db_id}" \ + -b /tmp/superset_cookies.txt \ + -H "${AUTH_HEADER}" \ + -H "${CSRF_HEADER}" \ + -H "Content-Type: application/json" \ + -d "{\"sqlalchemy_uri\": \"${uri}\"}") + http_code=$(echo "${response}" | tail -1) + if [ "${http_code}" = "200" ]; then + echo " Updated successfully." + else + echo " ERROR (HTTP ${http_code}): $(echo "${response}" | sed '$d')" + fi +} + +_register_db() { + local name="$1" uri="$2" + echo ">>> Registering: ${name}" + + # Check if connection already exists + local existing + existing=$(curl -s \ + -H "${AUTH_HEADER}" \ + "${SUPERSET_URL}/api/v1/database/?q=(filters:!((col:database_name,opr:DatabaseFilter,val:'${name}')))" \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('count',0))" 2>/dev/null || echo "0") + + if [ "${existing}" -gt 0 ] 2>/dev/null; then + echo " Already registered — skipping." + return + fi + + local response + response=$(curl -s -w "\n%{http_code}" \ + -X POST "${SUPERSET_URL}/api/v1/database/" \ + -b /tmp/superset_cookies.txt \ + -H "${AUTH_HEADER}" \ + -H "${CSRF_HEADER}" \ + -H "Content-Type: application/json" \ + -d "{ + \"database_name\": \"${name}\", + \"sqlalchemy_uri\": \"${uri}\", + \"expose_in_sqllab\": true, + \"allow_run_async\": false + }") + + local http_code body + http_code=$(echo "${response}" | tail -1) + body=$(echo "${response}" | sed '$d') + + if [ "${http_code}" = "201" ]; then + echo " Registered successfully." + else + echo " ERROR (HTTP ${http_code}): ${body}" + fi +} + +# ── wait for Superset ───────────────────────────────────────────────────────── +echo ">>> Waiting for Superset to be ready..." +until curl -sf "${SUPERSET_URL}/health" > /dev/null; do + sleep 3 +done +echo ">>> Superset is up." + +# ── authenticate ────────────────────────────────────────────────────────────── +LOGIN_RESPONSE=$(curl -s -c /tmp/superset_cookies.txt \ + -X POST "${SUPERSET_URL}/api/v1/security/login" \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"${ADMIN_USER}\", \"password\": \"${ADMIN_PASS}\", \"provider\": \"db\", \"refresh\": true}") + +ACCESS_TOKEN=$(echo "${LOGIN_RESPONSE}" | python3 -c "import sys, json; print(json.load(sys.stdin)['access_token'])") +echo ">>> Authenticated." + +AUTH_HEADER="Authorization: Bearer ${ACCESS_TOKEN}" + +# Fetch CSRF token (required for all POST requests). +# Must use the same session cookie from login — otherwise Superset sees a mismatched session. +CSRF_TOKEN=$(curl -s \ + -b /tmp/superset_cookies.txt \ + -c /tmp/superset_cookies.txt \ + -H "${AUTH_HEADER}" \ + "${SUPERSET_URL}/api/v1/security/csrf_token/" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['result'])") +CSRF_HEADER="X-CSRFToken: ${CSRF_TOKEN}" +echo ">>> CSRF token obtained." + +# ── Snowflake connection ─────────────────────────────────────────────────────── +# Build the account identifier: use ORG-ACCOUNT format if both vars are set, +# otherwise fall back to SNOWFLAKE_ACCOUNT as-is (may already contain the full identifier) +if [ -n "${SNOWFLAKE_ORG:-}" ]; then + SF_ACCOUNT="${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" +else + SF_ACCOUNT="${SNOWFLAKE_ACCOUNT}" +fi + +# URL-encode the password so special characters (#, !, *, @, etc.) don't break the URI +SF_PASSWORD_ENC=$(python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "${SNOWFLAKE_PASSWORD}") +SF_USER_ENC=$(python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "${SNOWFLAKE_USER}") + +SNOWFLAKE_URI="snowflake://${SF_USER_ENC}:${SF_PASSWORD_ENC}@${SF_ACCOUNT}/NYC_TAXI_DB/ANALYTICS?warehouse=ANALYTICS_WH&role=ANALYST_ROLE" +_register_db "NYC Taxi — Snowflake (Source)" "${SNOWFLAKE_URI}" + +# ── ClickHouse connection (pre-configured for Act 2) ───────────────────────── +# Only register if CLICKHOUSE_HOST is explicitly set — it won't be available until Act 2 +if [ -n "${CLICKHOUSE_HOST:-}" ]; then + CH_PASSWORD_ENC=$(python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "${CLICKHOUSE_PASSWORD:-}") + CLICKHOUSE_URI="clickhousedb://${CLICKHOUSE_USER:-default}:${CH_PASSWORD_ENC}@${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT:-8443}/nyc_taxi?protocol=https" + _register_db "NYC Taxi — ClickHouse Cloud (Target)" "${CLICKHOUSE_URI}" +else + echo ">>> Skipping ClickHouse connection (CLICKHOUSE_HOST not set — configure in Act 2)." +fi + +# ── Import dashboards ───────────────────────────────────────────────────────── +# Superset expects dashboard exports as .zip files. +# The stub .json files in dashboards/ are metadata placeholders — not importable. +# After building dashboards manually in the UI, export them as .zip and place +# them in the dashboards/ directory, then re-run this script to import them. +echo ">>> Importing dashboards..." +shopt -s nullglob +dashboard_files=("${SCRIPT_DIR}/dashboards"/*.zip) +if [ ${#dashboard_files[@]} -eq 0 ]; then + echo " No dashboard exports found in dashboards/ (expected .zip files)." + echo " Build dashboards in the UI, export them, then re-run this script." +else + for dashboard_file in "${dashboard_files[@]}"; do + dashboard_name=$(basename "${dashboard_file}" .zip) + echo " Importing: ${dashboard_name}" + + # The exported zip has database passwords redacted as XXXXXXXXXX. + # Build a passwords JSON mapping each databases/*.yaml path in the zip + # to the real Snowflake password so Superset can validate and import. + # Superset strips the zip root directory before matching passwords. + # Keys must be root-stripped paths: "databases/Foo.yaml" not "export_dir/databases/Foo.yaml" + PASSWORDS=$(python3 -c " +import zipfile, json, sys +zip_path, sf_pass = sys.argv[1], sys.argv[2] +pw = {} +with zipfile.ZipFile(zip_path) as z: + for name in z.namelist(): + if '/databases/' in name and name.endswith('.yaml'): + stripped = '/'.join(name.split('/')[1:]) + pw[stripped] = sf_pass +print(json.dumps(pw)) +" "${dashboard_file}" "${SNOWFLAKE_PASSWORD}") + + response=$(curl -s -w "\n%{http_code}" \ + -X POST "${SUPERSET_URL}/api/v1/dashboard/import/" \ + -b /tmp/superset_cookies.txt \ + -H "${AUTH_HEADER}" \ + -H "${CSRF_HEADER}" \ + -F "formData=@${dashboard_file};type=application/zip" \ + -F "overwrite=true" \ + -F "passwords=${PASSWORDS}") + http_code=$(echo "${response}" | tail -1) + if [ "${http_code}" = "200" ]; then + echo " Imported successfully." + else + echo " WARNING (HTTP ${http_code}): $(echo "${response}" | sed '$d')" + fi + done +fi + +# ── Re-apply correct connection URIs (dashboard import may overwrite them) ──── +# Dashboard ZIPs embed the exporter's credentials. Re-stamp with env-var values +# so the connection always reflects the current environment, regardless of import order. +_update_db "NYC Taxi — Snowflake (Source)" "${SNOWFLAKE_URI}" +if [ -n "${CLICKHOUSE_HOST:-}" ]; then + _update_db "NYC Taxi — ClickHouse Cloud (Target)" "${CLICKHOUSE_URI}" +fi + +echo "" +echo "============================================================" +echo " Superset initialized." +echo " URL: ${SUPERSET_URL}" +echo " Username: ${ADMIN_USER}" +echo " Password: ${ADMIN_PASS}" +echo "============================================================" diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/superset_config.py b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/superset_config.py new file mode 100644 index 0000000..5891051 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/superset/superset_config.py @@ -0,0 +1,46 @@ +import os + +# Database +SQLALCHEMY_DATABASE_URI = ( + f"postgresql+psycopg2://superset:{os.environ.get('SUPERSET_DB_PASSWORD', 'superset')}" + f"@superset_db:5432/superset" +) + +# Cache (Redis) +CACHE_CONFIG = { + "CACHE_TYPE": "RedisCache", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_", + "CACHE_REDIS_URL": "redis://superset_cache:6379/0", +} + +DATA_CACHE_CONFIG = { + "CACHE_TYPE": "RedisCache", + "CACHE_DEFAULT_TIMEOUT": 3600, + "CACHE_KEY_PREFIX": "superset_data_", + "CACHE_REDIS_URL": "redis://superset_cache:6379/1", +} + +# Security +SECRET_KEY = os.environ.get("SUPERSET_SECRET_KEY", "change-me-in-production-32chars!!") +WTF_CSRF_ENABLED = True +SESSION_COOKIE_HTTPONLY = True +SESSION_COOKIE_SECURE = False # Set True in production with HTTPS + +# Feature flags +FEATURE_FLAGS = { + "ENABLE_TEMPLATE_PROCESSING": True, + "DASHBOARD_NATIVE_FILTERS": True, + "DASHBOARD_CROSS_FILTERS": True, + "ALERT_REPORTS": False, +} + +# Allow Snowflake and ClickHouse database connections +PREVENT_UNSAFE_DB_CONNECTIONS = False + +# Row limit for query results +ROW_LIMIT = 50000 +VIZ_ROW_LIMIT = 10000 + +# Default dashboard refresh interval (seconds) +DEFAULT_DASHBOARD_REFRESH_FREQUENCY = 900 # 15 minutes diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/teardown.sh b/workshop_public/snowflake_migration_lab/01-setup-snowflake/teardown.sh new file mode 100755 index 0000000..cd88239 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/teardown.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# ============================================================ +# teardown.sh — Destroy all lab resources +# +# WARNING: This permanently deletes all Snowflake resources +# and stops all Docker containers. Run only when done with +# the lab or resetting for a new cohort. +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TERRAFORM_DIR="${SCRIPT_DIR}/terraform" +SUPERSET_DIR="${SCRIPT_DIR}/superset" + +# Auto-source .env so teardown works without manually running `source .env` +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + set -a + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.env" + set +a +fi + +log() { echo -e "\n\033[1;34m>>> $*\033[0m"; } +ok() { echo -e "\033[1;32m ✓ $*\033[0m"; } +warn() { echo -e "\033[1;33m ⚠ $*\033[0m"; } + +# Confirm +echo "" +echo " ╔════════════════════════════════════════════════════════╗" +echo " ║ WARNING: This will PERMANENTLY DELETE all Snowflake ║" +echo " ║ resources for the NYC Taxi lab environment. ║" +echo " ║ All data, tables, warehouses, and roles will be ║" +echo " ║ destroyed. This cannot be undone. ║" +echo " ╚════════════════════════════════════════════════════════╝" +echo "" +read -r -p " Type 'destroy' to confirm: " confirmation +if [[ "${confirmation}" != "destroy" ]]; then + echo " Teardown cancelled." + exit 0 +fi + +log "Stopping Apache Superset..." +if [[ -f "${SUPERSET_DIR}/docker-compose.yml" ]] && command -v docker >/dev/null 2>&1; then + cd "${SUPERSET_DIR}" + docker-compose down -v 2>/dev/null || true + ok "Superset stopped and volumes removed." + cd "${SCRIPT_DIR}" +else + warn "Superset not running or docker not found." +fi + +log "Zeroing Time Travel retention on NYC_TAXI_DB..." +# Set DATA_RETENTION_TIME_IN_DAYS = 0 before destroying so Snowflake does not +# hold dropped data in Time Travel (default 1 day) or surface it in the +# Horizon catalog after teardown. Must run BEFORE terraform destroy. +SNOWSQL_CMD="" +if command -v snowsql >/dev/null 2>&1; then SNOWSQL_CMD="snowsql" +elif [[ -x "/Applications/SnowSQL.app/Contents/MacOS/snowsql" ]]; then SNOWSQL_CMD="/Applications/SnowSQL.app/Contents/MacOS/snowsql" +fi + +if [[ -n "${SNOWSQL_CMD}" && -n "${SNOWFLAKE_PASSWORD:-}" ]]; then + SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename SYSADMIN \ + --option output_format=plain --option friendly=false \ + -q "ALTER DATABASE IF EXISTS NYC_TAXI_DB SET DATA_RETENTION_TIME_IN_DAYS = 0;" \ + 2>/dev/null || true + ok "Time Travel retention zeroed — data will not persist in Horizon catalog." +else + warn "snowsql not found or credentials missing — skipping Time Travel reset." + warn "Data may remain visible in Snowflake Horizon for up to 1 day." +fi + +log "Destroying Snowflake infrastructure with Terraform..." +cd "${TERRAFORM_DIR}" +if [[ -f "terraform.tfstate" ]]; then + terraform destroy -auto-approve -input=false + ok "All Snowflake resources destroyed." +else + warn "No Terraform state found. Resources may have already been destroyed." +fi +cd "${SCRIPT_DIR}" + +echo "" +echo "============================================================" +echo " Teardown complete. All Snowflake credits have stopped." +echo " The local dbt project, queries, and scripts remain intact." +echo "" +echo " Note: COMPUTE_WH is Snowflake's default account warehouse." +echo " It was not created by this lab and is not destroyed by" +echo " teardown. It will auto-suspend when idle (no credits lost)." +echo "============================================================" diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/backend.hcl.example b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/backend.hcl.example new file mode 100644 index 0000000..1710ec1 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/backend.hcl.example @@ -0,0 +1,12 @@ +# Optional: use S3 backend for shared state across team members. +# Steps: +# 1. Copy this file to backend.hcl (gitignored). +# 2. Fill in the values for your environment. +# 3. Run: terraform init -backend-config=backend.hcl +# 4. In main.tf, replace the backend "local" block with backend "s3" {}. + +bucket = "clickhouse-lab-tf-state" +key = "snowflake/migration-lab/terraform.tfstate" +region = "us-east-1" +encrypt = true +dynamodb_table = "clickhouse-lab-tf-lock" diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/database.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/database.tf new file mode 100644 index 0000000..0e1f6ec --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/database.tf @@ -0,0 +1,64 @@ +resource "snowflake_database" "nyc_taxi" { + name = "NYC_TAXI_DB" + comment = "NYC Taxi migration lab — source Snowflake environment" +} + +# Transfer database and schema ownership to SYSADMIN so SQL scripts and +# dbt (which use SYSADMIN-owned objects) have full DDL access. +resource "snowflake_grant_ownership" "nyc_taxi_db_to_sysadmin" { + account_role_name = "SYSADMIN" + on { + object_type = "DATABASE" + object_name = snowflake_database.nyc_taxi.name + } + outbound_privileges = "COPY" + depends_on = [snowflake_database.nyc_taxi] +} + +resource "snowflake_schema" "raw" { + database = snowflake_database.nyc_taxi.name + name = "RAW" + comment = "Raw ingested data — immutable source of truth" +} + +resource "snowflake_grant_ownership" "raw_schema_to_sysadmin" { + account_role_name = "SYSADMIN" + outbound_privileges = "COPY" + on { + object_type = "SCHEMA" + object_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + depends_on = [snowflake_schema.raw] +} + +resource "snowflake_schema" "staging" { + database = snowflake_database.nyc_taxi.name + name = "STAGING" + comment = "dbt staging models — cleaned and typed" +} + +resource "snowflake_grant_ownership" "staging_schema_to_sysadmin" { + account_role_name = "SYSADMIN" + outbound_privileges = "COPY" + on { + object_type = "SCHEMA" + object_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } + depends_on = [snowflake_schema.staging] +} + +resource "snowflake_schema" "analytics" { + database = snowflake_database.nyc_taxi.name + name = "ANALYTICS" + comment = "Dimensional model — served to BI tools" +} + +resource "snowflake_grant_ownership" "analytics_schema_to_sysadmin" { + account_role_name = "SYSADMIN" + outbound_privileges = "COPY" + on { + object_type = "SCHEMA" + object_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + depends_on = [snowflake_schema.analytics] +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/main.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/main.tf new file mode 100644 index 0000000..2cd7836 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/main.tf @@ -0,0 +1,40 @@ +terraform { + required_version = ">= 1.6.0" + required_providers { + snowflake = { + source = "snowflakedb/snowflake" + version = ">= 2.0.0" + } + } + + backend "local" { + path = "terraform.tfstate" + } + + # S3 backend (uncomment and remove backend "local" block above to use): + # backend "s3" { + # bucket = "clickhouse-lab-tf-state" + # key = "snowflake/migration-lab/terraform.tfstate" + # region = "us-east-1" + # } +} + +# Authentication note: +# For CI/CD pipelines, replace `password` with `private_key_path`: +# openssl genrsa 2048 | openssl pkcs8 -topk8 -inform PEM -out snowflake_rsa_key.p8 -nocrypt +# openssl rsa -in snowflake_rsa_key.p8 -pubout -out snowflake_rsa_key.pub +# ALTER USER TERRAFORM_SVC SET RSA_PUBLIC_KEY=''; + +# ACCOUNTADMIN is required to: +# - Create account-level roles (SECURITYADMIN privilege) +# - Create resource monitors (ACCOUNTADMIN-only) +# - Grant roles to users (MANAGE GRANTS privilege) +# Database objects (database, schema, warehouse) are created by ACCOUNTADMIN and +# immediately transferred to SYSADMIN ownership so SQL scripts work as SYSADMIN. +provider "snowflake" { + organization_name = var.snowflake_org + account_name = var.snowflake_account + user = var.snowflake_user + password = var.snowflake_password # use private_key_path for CI/CD + role = "ACCOUNTADMIN" +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/monitors.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/monitors.tf new file mode 100644 index 0000000..b75eee0 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/monitors.tf @@ -0,0 +1,14 @@ +resource "snowflake_resource_monitor" "analytics_wh_monitor" { + name = "ANALYTICS_WH_MONITOR" + credit_quota = 50 + frequency = "MONTHLY" + + # IMMEDIATELY means the monitor starts tracking from the moment it is created. + start_timestamp = "IMMEDIATELY" + + # Send an email notification to account admins at 75% consumption. + notify_triggers = [75] + + # Suspend the warehouse (but allow current queries to finish) at 100% consumption. + suspend_trigger = 100 +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/outputs.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/outputs.tf new file mode 100644 index 0000000..9b8c73b --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/outputs.tf @@ -0,0 +1,44 @@ +output "database_name" { + description = "Name of the NYC Taxi migration database." + value = snowflake_database.nyc_taxi.name +} + +output "transform_warehouse_name" { + description = "Name of the transformation warehouse used by dbt and ELT pipelines." + value = snowflake_warehouse.transform_wh.name +} + +output "analytics_warehouse_name" { + description = "Name of the analytics warehouse used by BI tools and ad-hoc queries." + value = snowflake_warehouse.analytics_wh.name +} + +output "raw_schema" { + description = "Fully-qualified name of the RAW schema." + value = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" +} + +output "staging_schema" { + description = "Fully-qualified name of the STAGING schema." + value = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" +} + +output "analytics_schema" { + description = "Fully-qualified name of the ANALYTICS schema." + value = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" +} + +output "connection_info" { + description = "Human-readable summary of the provisioned Snowflake environment." + value = <<-EOT + Snowflake Environment — ${var.environment} / cohort: ${var.lab_cohort} + --------------------------------------------------------------- + Organization : ${var.snowflake_org} + Account : ${var.snowflake_account} + Database : ${snowflake_database.nyc_taxi.name} + Schemas : ${snowflake_schema.raw.name}, ${snowflake_schema.staging.name}, ${snowflake_schema.analytics.name} + Warehouses : ${snowflake_warehouse.transform_wh.name} (SMALL), ${snowflake_warehouse.analytics_wh.name} (MEDIUM) + Roles : ${snowflake_account_role.transformer.name}, ${snowflake_account_role.analyst.name}, ${snowflake_account_role.dbt.name}, ${snowflake_account_role.loader.name} + Resource Monitor: ${snowflake_resource_monitor.analytics_wh_monitor.name} + EOT +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/roles.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/roles.tf new file mode 100644 index 0000000..bb9f76f --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/roles.tf @@ -0,0 +1,403 @@ +# --------------------------------------------------------------------------- +# Role definitions (provider v2.x: snowflake_account_role) +# --------------------------------------------------------------------------- + +resource "snowflake_account_role" "transformer" { + name = "TRANSFORMER_ROLE" + comment = "Runs ELT pipelines and dbt staging models — lab cohort: ${var.lab_cohort}" +} + +resource "snowflake_account_role" "analyst" { + name = "ANALYST_ROLE" + comment = "Read-only access to the ANALYTICS schema for BI tools — lab cohort: ${var.lab_cohort}" +} + +resource "snowflake_account_role" "dbt" { + name = "DBT_ROLE" + comment = "Full dbt service account role — reads RAW, writes STAGING and ANALYTICS — lab cohort: ${var.lab_cohort}" +} + +resource "snowflake_account_role" "loader" { + name = "LOADER_ROLE" + comment = "Ingestion service account role — loads raw data into RAW schema — lab cohort: ${var.lab_cohort}" +} + +# --------------------------------------------------------------------------- +# SYSADMIN ownership of custom roles +# --------------------------------------------------------------------------- + +resource "snowflake_grant_account_role" "sysadmin_owns_transformer" { + role_name = snowflake_account_role.transformer.name + parent_role_name = "SYSADMIN" +} + +resource "snowflake_grant_account_role" "sysadmin_owns_analyst" { + role_name = snowflake_account_role.analyst.name + parent_role_name = "SYSADMIN" +} + +resource "snowflake_grant_account_role" "sysadmin_owns_dbt" { + role_name = snowflake_account_role.dbt.name + parent_role_name = "SYSADMIN" +} + +resource "snowflake_grant_account_role" "sysadmin_owns_loader" { + role_name = snowflake_account_role.loader.name + parent_role_name = "SYSADMIN" +} + +# --------------------------------------------------------------------------- +# Grant all lab roles to the lab user so they can switch roles in the UI +# and dbt / the producer can connect with the right role. +# Requires ACCOUNTADMIN (MANAGE GRANTS privilege). +# --------------------------------------------------------------------------- + +resource "snowflake_grant_account_role" "user_gets_transformer" { + role_name = snowflake_account_role.transformer.name + user_name = var.snowflake_user +} + +resource "snowflake_grant_account_role" "user_gets_analyst" { + role_name = snowflake_account_role.analyst.name + user_name = var.snowflake_user +} + +resource "snowflake_grant_account_role" "user_gets_dbt" { + role_name = snowflake_account_role.dbt.name + user_name = var.snowflake_user +} + +resource "snowflake_grant_account_role" "user_gets_loader" { + role_name = snowflake_account_role.loader.name + user_name = var.snowflake_user +} + + +# =========================================================================== +# TRANSFORMER_ROLE grants +# =========================================================================== + +resource "snowflake_grant_privileges_to_account_role" "transformer_database" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = snowflake_database.nyc_taxi.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_warehouse" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.transform_wh.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_raw_schema" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["USAGE", "CREATE TABLE", "CREATE VIEW"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_raw_tables" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["SELECT"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_raw_future_tables" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["SELECT"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_staging_schema" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["USAGE", "CREATE TABLE", "CREATE VIEW"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_staging_tables" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["SELECT"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "transformer_staging_future_tables" { + account_role_name = snowflake_account_role.transformer.name + privileges = ["SELECT"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } + } +} + +# =========================================================================== +# ANALYST_ROLE grants +# =========================================================================== + +resource "snowflake_grant_privileges_to_account_role" "analyst_database" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = snowflake_database.nyc_taxi.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "analyst_warehouse" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.analytics_wh.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "analyst_analytics_schema" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["USAGE"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "analyst_analytics_tables" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["SELECT"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "analyst_analytics_future_tables" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["SELECT"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "analyst_analytics_views" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["SELECT"] + on_schema_object { + all { + object_type_plural = "VIEWS" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "analyst_analytics_future_views" { + account_role_name = snowflake_account_role.analyst.name + privileges = ["SELECT"] + on_schema_object { + future { + object_type_plural = "VIEWS" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + } +} + +# =========================================================================== +# DBT_ROLE grants +# =========================================================================== + +resource "snowflake_grant_privileges_to_account_role" "dbt_database" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["USAGE", "CREATE SCHEMA"] + on_account_object { + object_type = "DATABASE" + object_name = snowflake_database.nyc_taxi.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_transform_warehouse" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.transform_wh.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_analytics_warehouse" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.analytics_wh.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_raw_schema" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["USAGE"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_raw_tables" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["SELECT"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_raw_future_tables" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["SELECT"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_staging_schema" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["USAGE", "CREATE TABLE", "CREATE VIEW", "CREATE STAGE"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_staging_tables" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_staging_future_tables" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.staging.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_analytics_schema" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["USAGE", "CREATE TABLE", "CREATE VIEW", "CREATE STAGE"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_analytics_tables" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "dbt_analytics_future_tables" { + account_role_name = snowflake_account_role.dbt.name + privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "TRUNCATE"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.analytics.name}" + } + } +} + +# =========================================================================== +# LOADER_ROLE grants +# =========================================================================== + +resource "snowflake_grant_privileges_to_account_role" "loader_database" { + account_role_name = snowflake_account_role.loader.name + privileges = ["USAGE"] + on_account_object { + object_type = "DATABASE" + object_name = snowflake_database.nyc_taxi.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "loader_warehouse" { + account_role_name = snowflake_account_role.loader.name + privileges = ["USAGE"] + on_account_object { + object_type = "WAREHOUSE" + object_name = snowflake_warehouse.transform_wh.name + } +} + +resource "snowflake_grant_privileges_to_account_role" "loader_raw_schema" { + account_role_name = snowflake_account_role.loader.name + privileges = ["USAGE", "CREATE TABLE"] + on_schema { + schema_name = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } +} + +resource "snowflake_grant_privileges_to_account_role" "loader_raw_tables" { + account_role_name = snowflake_account_role.loader.name + privileges = ["INSERT", "SELECT"] + on_schema_object { + all { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + } +} + +resource "snowflake_grant_privileges_to_account_role" "loader_raw_future_tables" { + account_role_name = snowflake_account_role.loader.name + privileges = ["INSERT", "SELECT"] + on_schema_object { + future { + object_type_plural = "TABLES" + in_schema = "${snowflake_database.nyc_taxi.name}.${snowflake_schema.raw.name}" + } + } +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/streams.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/streams.tf new file mode 100644 index 0000000..04af5b7 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/streams.tf @@ -0,0 +1,103 @@ +# Streams and tasks are applied in a targeted second `terraform apply` inside setup.sh, +# AFTER 01_create_tables.sql has created TRIPS_RAW. They cannot be part of the initial +# apply because the stream requires the table to exist first. +# +# setup.sh handles the correct ordering automatically. +# To apply manually: +# terraform apply \ +# -target=snowflake_execute.trips_cdc_stream \ +# -target=snowflake_execute.cdc_consume_task \ +# -target=snowflake_execute.hourly_agg_task \ +# -auto-approve + +# --------------------------------------------------------------------------- +# CDC stream on TRIPS_RAW +# Captures all DML changes (INSERT/UPDATE/DELETE). +# In the migration lab, this stream is retired at cutover — live writes go directly to ClickHouse. +# --------------------------------------------------------------------------- +resource "snowflake_execute" "trips_cdc_stream" { + execute = <<-SQL + CREATE OR REPLACE STREAM NYC_TAXI_DB.RAW.TRIPS_CDC_STREAM + ON TABLE NYC_TAXI_DB.RAW.TRIPS_RAW + APPEND_ONLY = FALSE + COMMENT = 'CDC stream for incremental ClickHouse sync — migration lab'; + SQL + revert = "DROP STREAM IF EXISTS NYC_TAXI_DB.RAW.TRIPS_CDC_STREAM" + depends_on = [snowflake_schema.raw] +} + +# --------------------------------------------------------------------------- +# Task 1: consume CDC stream every 5 minutes +# Created SUSPENDED — resume after setup: +# ALTER TASK NYC_TAXI_DB.RAW.CDC_CONSUME_TASK RESUME; +# +# In the migration lab this query shows the source-side CDC mechanism. +# Migration challenge: Snowflake Tasks have no native equivalent in +# ClickHouse — replaced by the post-cutover producer (direct writes) or Debezium for real CDC. +# --------------------------------------------------------------------------- +resource "snowflake_execute" "cdc_consume_task" { + execute = <<-SQL + CREATE OR REPLACE TASK NYC_TAXI_DB.RAW.CDC_CONSUME_TASK + WAREHOUSE = TRANSFORM_WH + SCHEDULE = 'USING CRON */5 * * * * UTC' + COMMENT = 'Consumes TRIPS_CDC_STREAM every 5 min — retired at cutover in migration lab' + AS + SELECT + METADATA$ACTION AS cdc_action, + METADATA$ISUPDATE AS is_update, + METADATA$ROW_ID AS row_id, + TRIP_ID, + PICKUP_DATETIME, + TOTAL_AMOUNT, + TRIP_METADATA + FROM NYC_TAXI_DB.RAW.TRIPS_CDC_STREAM + WHERE METADATA$ACTION = 'INSERT' + ORDER BY PICKUP_DATETIME; + SQL + revert = "DROP TASK IF EXISTS NYC_TAXI_DB.RAW.CDC_CONSUME_TASK" + depends_on = [snowflake_execute.trips_cdc_stream] +} + +# --------------------------------------------------------------------------- +# Task 2: refresh hourly zone aggregates via MERGE (runs every hour) +# Created SUSPENDED — resume after dbt run: +# ALTER TASK NYC_TAXI_DB.STAGING.HOURLY_AGG_TASK RESUME; +# +# Migration challenge: ClickHouse has no MERGE statement. +# ClickHouse equivalent: ReplacingMergeTree + INSERT (no explicit MERGE needed). +# --------------------------------------------------------------------------- +resource "snowflake_execute" "hourly_agg_task" { + execute = <<-SQL + CREATE OR REPLACE TASK NYC_TAXI_DB.STAGING.HOURLY_AGG_TASK + WAREHOUSE = TRANSFORM_WH + SCHEDULE = 'USING CRON 0 * * * * UTC' + COMMENT = 'Refreshes AGG_HOURLY_ZONE_TRIPS every hour via MERGE — migration challenge: no MERGE in ClickHouse' + AS + MERGE INTO NYC_TAXI_DB.ANALYTICS.AGG_HOURLY_ZONE_TRIPS AS target + USING ( + SELECT + DATE_TRUNC('hour', pickup_at) AS hour_bucket, + pickup_location_id AS zone_id, + COUNT(*) AS trips, + SUM(total_amount_usd) AS revenue, + AVG(trip_distance_miles) AS avg_distance + FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS + WHERE pickup_at >= DATEADD('hour', -2, CURRENT_TIMESTAMP()) + GROUP BY 1, 2 + ) AS source + ON target.hour_bucket = source.hour_bucket + AND target.zone_id = source.zone_id + WHEN MATCHED THEN UPDATE SET + target.trips = source.trips, + target.revenue = source.revenue, + target.avg_distance = source.avg_distance, + target.updated_at = CURRENT_TIMESTAMP() + WHEN NOT MATCHED THEN INSERT + (hour_bucket, zone_id, trips, revenue, avg_distance) + VALUES + (source.hour_bucket, source.zone_id, source.trips, + source.revenue, source.avg_distance); + SQL + revert = "DROP TASK IF EXISTS NYC_TAXI_DB.STAGING.HOURLY_AGG_TASK" + depends_on = [snowflake_execute.trips_cdc_stream] +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/terraform.tfvars.example b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/terraform.tfvars.example new file mode 100644 index 0000000..1c56899 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/terraform.tfvars.example @@ -0,0 +1,13 @@ +# Copy to terraform.tfvars and fill in your values. +# Never commit terraform.tfvars to source control — add it to .gitignore. + +snowflake_org = "MYORG" # e.g. CLICKHOUSE +snowflake_account = "MYACCOUNT" # e.g. ABC12345 +snowflake_user = "TERRAFORM_SVC" + +# Choose one authentication method: +snowflake_password = "your-password-here" +# snowflake_private_key_path = "~/.ssh/snowflake_rsa_key.p8" + +environment = "lab" +lab_cohort = "fy27-q1" diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/variables.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/variables.tf new file mode 100644 index 0000000..f65a083 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/variables.tf @@ -0,0 +1,40 @@ +variable "snowflake_org" { + description = "Snowflake organization name (the part before the account in the account identifier)." + type = string +} + +variable "snowflake_account" { + description = "Snowflake account name (the part after the organization in the account identifier)." + type = string +} + +variable "snowflake_user" { + description = "Snowflake user that Terraform authenticates as. Should be a dedicated service account." + type = string + default = "TERRAFORM_SVC" +} + +variable "snowflake_password" { + description = "Password for the Snowflake service account. Use an empty string when authenticating via private key." + type = string + sensitive = true + default = "" +} + +variable "snowflake_private_key_path" { + description = "Path to the PKCS#8 RSA private key file used for key-pair authentication. Leave empty when using password auth." + type = string + default = "" +} + +variable "environment" { + description = "Deployment environment label (e.g. lab, dev, prod). Used in comments and tags." + type = string + default = "lab" +} + +variable "lab_cohort" { + description = "Workshop cohort identifier embedded in resource comments for cost attribution and lifecycle tracking." + type = string + default = "fy27-q1" +} diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/warehouses.tf b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/warehouses.tf new file mode 100644 index 0000000..c4ad968 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/terraform/warehouses.tf @@ -0,0 +1,16 @@ +resource "snowflake_warehouse" "transform_wh" { + name = "TRANSFORM_WH" + warehouse_size = "SMALL" + auto_suspend = 60 + auto_resume = true + comment = "Used by dbt and ELT pipelines — lab cohort: ${var.lab_cohort}" +} + +resource "snowflake_warehouse" "analytics_wh" { + name = "ANALYTICS_WH" + warehouse_size = "SMALL" + auto_suspend = 60 + auto_resume = true + resource_monitor = snowflake_resource_monitor.analytics_wh_monitor.name + comment = "Used by BI tools and ad-hoc analyst queries — lab cohort: ${var.lab_cohort}" +} diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/migration-plan.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/migration-plan.md new file mode 100644 index 0000000..e1b02b8 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/migration-plan.md @@ -0,0 +1,231 @@ +# Migration Plan — NYC Taxi Workload + +**Partner:** _your name_ +**Date:** _today's date_ +**Source:** Snowflake — NYC_TAXI_DB +**Target:** ClickHouse Cloud + +--- + +## Completion Checklist + +Part 3's `setup.sh` reads these checkboxes. Change `[ ]` to `[x]` when each section is complete. + +- [ ] Engine selection: completed +- [ ] Sort key design: completed +- [ ] Schema translation: completed +- [ ] Migration wave plan: completed +- [ ] dbt model design: completed + +--- + +## Section 1: Profile Summary + +*Paste key numbers from `profile_report.md` here after running `scripts/01_profile_snowflake.sh`.* + +| Metric | Value | +|--------|-------| +| Total tables | | +| Total views | | +| Streams | | +| Tasks | | +| Total rows in TRIPS_RAW | | +| Date range | | +| VARIANT columns | | +| QUALIFY usages detected | | +| MERGE INTO usages detected | | + +--- + +## Section 2: Object Inventory + +*List every object being migrated with its complexity grade (A/B/C/D).* + +| Object | Type | Schema | Rows | Complexity Grade | Notes | +|--------|------|--------|------|-----------------|-------| +| `trips_raw` | Table | raw | ~50M | | | +| `stg_trips` | dbt View | staging | — | | | +| `stg_taxi_zones` | dbt View | staging | — | | | +| `int_trips_enriched` | dbt Ephemeral | staging | — | | | +| `fact_trips` | dbt Incremental | analytics | ~50M | | | +| `agg_hourly_zone_trips` | dbt Incremental | analytics | | | | +| `dim_taxi_zones` | dbt Table | analytics | 265 | | | +| `dim_payment_type` | dbt Table | analytics | 6 | | | +| `dim_vendor` | dbt Table | analytics | 3 | | | +| `taxi_zones_dict` | Dictionary | analytics | 265 | | | +| `mv_hourly_revenue` | Refreshable MV | analytics | — | | | +| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | Snowflake Stream + Task | — | — | | | + +--- + +## Section 3: Engine Selection Decisions + +*Completed from Worksheet 1.* + +| Table | Engine | Version Column | Reasoning | +|-------|--------|---------------|-----------| +| `trips_raw` | | | | +| `fact_trips` | | | | +| `agg_hourly_zone_trips` | | | | +| `dim_taxi_zones` | | — | | +| `dim_payment_type` | | — | | +| `dim_vendor` | | — | | +| `mv_hourly_revenue` | | — | | + +--- + +## Section 4: Sort Key Design + +*Completed from Worksheet 2.* + +| Table | ORDER BY | Reasoning | +|-------|----------|-----------| +| `trips_raw` | | | +| `fact_trips` | | | +| `agg_hourly_zone_trips` | | | +| `dim_taxi_zones` | | | + +--- + +## Section 5: Schema Translation Notes + +*Completed from Worksheet 3. Record only non-obvious decisions.* + +| Column | Snowflake Type | ClickHouse Type | Decision Rationale | +|--------|---------------|----------------|-------------------| +| `TRIP_METADATA` | VARIANT | | | +| `PICKUP_DATETIME` | TIMESTAMP_NTZ(9) | | | +| `PICKUP_LOCATION_ID` | INTEGER | | | +| `VENDOR_ID` | INTEGER | | | +| `DRIVER_RATING` | FLOAT | | | +| `UPDATED_AT` | TIMESTAMP_NTZ(9) | | | + +### Function Translations Required + +| Snowflake Expression | ClickHouse Equivalent | +|----------------------|-----------------------| +| `DATE_TRUNC('hour', ...)` | | +| `DATEADD('day', -7, CURRENT_DATE)` | | +| `DATEDIFF('minute', t1, t2)` | | +| `TRIP_METADATA:driver.rating::FLOAT` | | +| `QUALIFY ROW_NUMBER() OVER (...) <= n` | | +| `MERGE INTO ... WHEN MATCHED THEN UPDATE` | | + +--- + +## Section 6: Migration Waves + +*Completed from Worksheet 4.* + +| Wave | Objects | Dependencies | Notes | +|------|---------|-------------|-------| +| Wave 0 | | | | +| Wave 1 | | | | +| Wave 2 | | | | +| Wave 3 | | | | +| Wave 4 | | | | + +### Risk Register (Grade C/D objects) + +| Object | Risk | Verification Method | +|--------|------|---------------------| +| | | | + +--- + +## Section 7: Known Dialect Gaps + +*Check all that apply to this workload.* + +- [ ] QUALIFY — affects: _list queries_ +- [ ] VARIANT colon-path — affects: _list queries_ +- [ ] LATERAL FLATTEN — affects: _list queries_ +- [ ] MERGE INTO — affects: _list dbt models_ +- [ ] Snowflake Streams / Tasks → producer cutover + Refreshable MVs +- [ ] Date function differences — affects: _list queries_ + +For each checked item, confirm the ClickHouse equivalent is documented in Section 5. + +--- + +## Section 8: Migration Strategy + +*Pre-selected for this lab. Annotate your understanding of why.* + +**Data movement:** Python migration script (`scripts/02_migrate_trips.py`) +- Bulk load for initial 50M rows via direct Snowflake → ClickHouse connection +- Resumable with `--resume` flag if interrupted +- Post-migration: producer cutover (`scripts/03_cutover.sh`) switches live writes directly to ClickHouse Cloud + +Why Python script over object storage relay or ClickPipes? + +*Your answer:* + +**Incremental strategy (dbt):** `delete_insert` + +Why `delete_insert` over `append` or `merge` strategy? + +*Your answer:* + +--- + +## Section 9: Cutover Criteria + +*These are the minimum requirements before declaring the migration complete. Pre-filled; verify you understand each threshold.* + +| Criterion | Threshold | Measured By | +|-----------|-----------|-------------| +| Row count parity | ≥ 99.9% match (CH ≥ SF post-cutover is expected) | `scripts/01_verify_migration.sh` | +| Checksum parity | MD5 match on 10K-row sample | `scripts/02_validate_parity.sql` | +| dbt test pass rate | 100% | `dbt test` in `dbt/nyc_taxi_dbt_ch` | +| Query result parity | All 7 queries return same results (within floating-point tolerance) | Manual comparison | + +--- + +--- + +## Section 10: dbt Model Design + +*Completed from Worksheet 5.* + +### Materialization Selection + +| Model | Materialization | Why | +|-------|-----------------|-----| +| `stg_trips` | | | +| `stg_taxi_zones` | | | +| `int_trips_enriched` | | | +| `fact_trips` | | | +| `agg_hourly_zone_trips` | | | +| `dim_taxi_zones` | | | +| `dim_payment_type` | | | +| `dim_vendor` | | | + +### Engine Configuration + +| Model | ENGINE | Version Column | Why | +|-------|--------|----------------|-----| +| `fact_trips` | | | | +| `agg_hourly_zone_trips` | | | | +| `dim_taxi_zones` | | — | | +| `dim_payment_type` | | — | | +| `dim_vendor` | | — | | + +### Incremental Strategy + +| Model | `unique_key` | `incremental_strategy` | Incremental filter | Why this filter? | +|-------|-------------|----------------------|-------------------|-----------------| +| `fact_trips` | | | | | +| `agg_hourly_zone_trips` | | | | | + +### FINAL Placement + +| Model | FINAL in FROM clause? | Why? | +|-------|----------------------|------| +| `stg_trips` | | | +| `int_trips_enriched` | | | +| `fact_trips` | | | + +--- + +*When all five checkboxes at the top are checked, proceed to `03-migrate-to-clickhouse/`.* diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh b/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh new file mode 100755 index 0000000..4805906 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh @@ -0,0 +1,424 @@ +#!/usr/bin/env bash +# ============================================================ +# 01_profile_snowflake.sh — Profile the Part 1 Snowflake environment +# +# Generates profile_report.md with four sections: +# 1. Object Inventory (tables, views, streams, tasks) +# 2. Query Workload (top 10 queries by total elapsed time) +# 3. Table Statistics (row counts, date ranges, null rates) +# 4. Schema Compatibility Gaps (auto-detected migration challenges) +# +# Prerequisites: +# - snowsql installed and on PATH +# - Part 1 environment running +# - SNOWFLAKE_ORG, SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PASSWORD set +# +# Usage: +# source ../01-setup-snowflake/.env +# ./scripts/01_profile_snowflake.sh +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODULE_DIR="${SCRIPT_DIR}/.." +OUTPUT_FILE="${MODULE_DIR}/profile_report.md" + +BOLD="\033[1m"; RESET="\033[0m" +BLUE="\033[1;34m"; GREEN="\033[1;32m"; YELLOW="\033[1;33m"; RED="\033[1;31m" + +log() { echo -e "\n${BLUE}${BOLD}▶ $*${RESET}"; } +ok() { echo -e "${GREEN}${BOLD}✓ $*${RESET}"; } +warn() { echo -e "${YELLOW} ⚠ $*${RESET}"; } +die() { echo -e "\n${RED}${BOLD}✗ ERROR: $*${RESET}\n"; exit 1; } + +# ── Detect snowsql ──────────────────────────────────────────── +SNOWSQL="" +if command -v snowsql >/dev/null 2>&1; then + SNOWSQL="snowsql" +elif [[ -f "/Applications/SnowSQL.app/Contents/MacOS/snowsql" ]]; then + SNOWSQL="/Applications/SnowSQL.app/Contents/MacOS/snowsql" +else + die "snowsql not found. Install from https://docs.snowflake.com/en/user-guide/snowsql-install-config" +fi + +# ── Check required env vars ─────────────────────────────────── +for v in SNOWFLAKE_ORG SNOWFLAKE_ACCOUNT SNOWFLAKE_USER SNOWFLAKE_PASSWORD; do + [[ -z "${!v:-}" ]] && die "Missing required env var: $v. Source Part 1's .env first." +done + +SNOWFLAKE_ACCOUNT_ID="${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" + +# ── Helper: run SnowSQL query, return result ────────────────── +run_snowsql() { + local role="${1}"; local warehouse="${2}"; local query="${3}" + SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL}" \ + -a "${SNOWFLAKE_ACCOUNT_ID}" \ + -u "${SNOWFLAKE_USER}" \ + --rolename "${role}" \ + --warehouse "${warehouse}" \ + -q "${query}" \ + --option output_format=plain \ + --option friendly=false \ + --option timing=false \ + 2>/dev/null || echo "(query failed)" +} + +echo -e "\n${BLUE}${BOLD}Part 2: Snowflake Profiling Script${RESET}" +echo "────────────────────────────────────" +echo " Output: ${OUTPUT_FILE}" + +# ── Start writing report ────────────────────────────────────── +REPORT_DATE=$(date -u "+%Y-%m-%d %H:%M UTC") + +cat > "${OUTPUT_FILE}" <
> "${OUTPUT_FILE}" <<'MD' +## Section 1: Object Inventory + +### Tables and Views + +MD + +TABLES_RESULT=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + t.table_schema, + t.table_name, + t.table_type, + COALESCE(t.row_count, 0) AS row_count, + COALESCE(t.bytes, 0) AS size_bytes, + CASE + WHEN t.row_count > 10000000 THEN 'C - Large (>10M rows)' + WHEN t.row_count > 100000 THEN 'B - Medium (100K-10M rows)' + WHEN t.row_count > 0 THEN 'A - Small (<100K rows)' + ELSE 'D - Unknown / empty' + END AS complexity_grade, + t.comment +FROM NYC_TAXI_DB.INFORMATION_SCHEMA.TABLES t +WHERE t.table_schema IN ('RAW', 'STAGING', 'ANALYTICS') +ORDER BY t.table_schema, t.table_type, t.table_name; +") + +echo "${TABLES_RESULT}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +# Check for VARIANT columns +cat >> "${OUTPUT_FILE}" <<'MD' + +### VARIANT Columns (require JSONExtract* translation) + +MD + +VARIANT_RESULT=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + c.table_schema, + c.table_name, + c.column_name, + c.data_type +FROM NYC_TAXI_DB.INFORMATION_SCHEMA.COLUMNS c +WHERE c.table_schema IN ('RAW', 'STAGING', 'ANALYTICS') + AND c.data_type = 'VARIANT' +ORDER BY c.table_schema, c.table_name, c.column_name; +") + +echo "${VARIANT_RESULT}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +# Streams and Tasks (requires ACCOUNTADMIN) +cat >> "${OUTPUT_FILE}" <<'MD' + +### Streams and Tasks + +> Note: SHOW STREAMS and SHOW TASKS require ACCOUNTADMIN. If the output below is empty, +> run these commands manually in the Snowflake UI: +> SHOW STREAMS IN DATABASE NYC_TAXI_DB; +> SHOW TASKS IN DATABASE NYC_TAXI_DB; + +MD + +STREAMS_RESULT=$(run_snowsql "ACCOUNTADMIN" "ANALYTICS_WH" "SHOW STREAMS IN DATABASE NYC_TAXI_DB;" 2>/dev/null || echo "(requires ACCOUNTADMIN — run manually in Snowflake UI)") +echo "**Streams:**" >> "${OUTPUT_FILE}" +echo "${STREAMS_RESULT}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +TASKS_RESULT=$(run_snowsql "ACCOUNTADMIN" "ANALYTICS_WH" "SHOW TASKS IN DATABASE NYC_TAXI_DB;" 2>/dev/null || echo "(requires ACCOUNTADMIN — run manually in Snowflake UI)") +echo "**Tasks:**" >> "${OUTPUT_FILE}" +echo "${TASKS_RESULT}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +ok "Section 1 written" + +# ── Section 2: Query Workload ───────────────────────────────── +log "Section 2: Query Workload (ACCOUNT_USAGE — may have 1-3hr lag)" + +cat >> "${OUTPUT_FILE}" <<'MD' + +--- + +## Section 2: Query Workload + +Top 10 queries by total elapsed time over the last 7 days. + +> If ACCOUNT_USAGE is unavailable (requires ACCOUNTADMIN + 1-3hr propagation delay), +> run `scripts/02_query_history.sql` manually in the Snowflake UI and paste results here. + +MD + +QUERY_HISTORY=$(run_snowsql "ACCOUNTADMIN" "ANALYTICS_WH" " +SELECT + LEFT(query_text, 200) AS query_preview, + execution_status, + COUNT(*) AS executions, + ROUND(AVG(total_elapsed_time)) AS avg_ms, + ROUND(MAX(total_elapsed_time)) AS max_ms, + ROUND(SUM(bytes_scanned) / 1024 / 1024 / 1024, 2) AS total_gb_scanned +FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY +WHERE database_name = 'NYC_TAXI_DB' + AND start_time >= DATEADD(day, -7, CURRENT_TIMESTAMP()) + AND execution_status = 'SUCCESS' + AND query_type NOT IN ('SHOW', 'DESCRIBE', 'USE', 'SET') +GROUP BY 1, 2 +ORDER BY SUM(total_elapsed_time) DESC +LIMIT 10; +" 2>/dev/null || echo "(ACCOUNT_USAGE unavailable — run scripts/02_query_history.sql in Snowflake UI)") + +echo "${QUERY_HISTORY}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +ok "Section 2 written" + +# ── Section 3: Table Statistics ─────────────────────────────── +log "Section 3: Table Statistics" + +cat >> "${OUTPUT_FILE}" <<'MD' + +--- + +## Section 3: Table Statistics + +MD + +# trips_raw stats +cat >> "${OUTPUT_FILE}" <<'MD' +### RAW.TRIPS_RAW + +MD + +TRIPS_RAW_STATS=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + COUNT(*) AS total_rows, + MIN(pickup_datetime) AS earliest_pickup, + MAX(pickup_datetime) AS latest_pickup, + ROUND(100.0 * SUM(CASE WHEN trip_id IS NULL THEN 1 ELSE 0 END) / COUNT(*), 2) AS pct_null_trip_id, + ROUND(100.0 * SUM(CASE WHEN pickup_datetime IS NULL THEN 1 ELSE 0 END) / COUNT(*), 2) AS pct_null_pickup, + ROUND(100.0 * SUM(CASE WHEN trip_metadata IS NULL THEN 1 ELSE 0 END) / COUNT(*), 2) AS pct_null_metadata, + ROUND(100.0 * SUM(CASE WHEN trip_metadata IS NOT NULL AND trip_metadata != 'null' THEN 1 ELSE 0 END) / COUNT(*), 2) AS pct_populated_variant, + COUNT(DISTINCT vendor_id) AS distinct_vendors, + COUNT(DISTINCT pickup_location_id) AS distinct_pickup_zones +FROM NYC_TAXI_DB.RAW.TRIPS_RAW; +") + +echo "${TRIPS_RAW_STATS}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +# fact_trips stats +cat >> "${OUTPUT_FILE}" <<'MD' + +### ANALYTICS.FACT_TRIPS + +MD + +FACT_TRIPS_STATS=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + COUNT(*) AS total_rows, + MIN(pickup_at) AS earliest_pickup, + MAX(pickup_at) AS latest_pickup, + ROUND(AVG(fare_amount), 2) AS avg_fare, + ROUND(AVG(trip_distance), 2) AS avg_distance, + ROUND(100.0 * SUM(CASE WHEN driver_rating IS NULL THEN 1 ELSE 0 END) / COUNT(*), 2) AS pct_null_driver_rating +FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS; +") + +echo "${FACT_TRIPS_STATS}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +# agg_hourly stats +cat >> "${OUTPUT_FILE}" <<'MD' + +### ANALYTICS.AGG_HOURLY_ZONE_TRIPS + +MD + +AGG_STATS=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + COUNT(*) AS total_rows, + MIN(hour_bucket) AS earliest_bucket, + MAX(hour_bucket) AS latest_bucket, + COUNT(DISTINCT zone_id) AS distinct_zones, + ROUND(SUM(trip_count)) AS total_trips_recorded +FROM NYC_TAXI_DB.ANALYTICS.AGG_HOURLY_ZONE_TRIPS; +") + +echo "${AGG_STATS}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +ok "Section 3 written" + +# ── Section 4: Schema Compatibility Gaps ───────────────────── +log "Section 4: Schema Compatibility Gaps" + +cat >> "${OUTPUT_FILE}" <<'MD' + +--- + +## Section 4: Schema Compatibility Gaps + +Auto-detected patterns that require migration attention. +Each gap maps to a worksheet or reference document for remediation. + +MD + +# Check for QUALIFY in stored procedures / views +cat >> "${OUTPUT_FILE}" <<'MD' +### Gap 1: QUALIFY Clauses + +Searching view definitions for QUALIFY usage... + +MD + +QUALIFY_CHECK=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + table_schema, + table_name, + 'Contains QUALIFY clause' AS gap_type, + 'Rewrite as subquery — see worksheets/03_schema_translation.md' AS remediation +FROM NYC_TAXI_DB.INFORMATION_SCHEMA.VIEWS +WHERE UPPER(view_definition) LIKE '%QUALIFY%' +ORDER BY table_schema, table_name; +") + +echo "${QUALIFY_CHECK}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +cat >> "${OUTPUT_FILE}" <<'MD' + +> Additionally, Q3 in `01-setup-snowflake/queries/` uses QUALIFY explicitly. +> ClickHouse has no QUALIFY — rewrite as subquery wrapping ROW_NUMBER(). +> See: https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Gap 1. + +### Gap 2: VARIANT Columns + +MD + +VARIANT_GAPS=$(run_snowsql "ANALYST_ROLE" "ANALYTICS_WH" " +SELECT + c.table_schema, + c.table_name, + c.column_name, + 'VARIANT column' AS gap_type, + 'Store as String in ClickHouse; use JSONExtract* at query time' AS remediation +FROM NYC_TAXI_DB.INFORMATION_SCHEMA.COLUMNS c +WHERE c.table_schema IN ('RAW', 'STAGING', 'ANALYTICS') + AND c.data_type = 'VARIANT' +ORDER BY c.table_schema, c.table_name; +") + +echo "${VARIANT_GAPS}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +cat >> "${OUTPUT_FILE}" <<'MD' + +### Gap 3: Snowflake Streams (CDC) + +MD + +STREAM_GAPS=$(run_snowsql "ACCOUNTADMIN" "ANALYTICS_WH" " +SELECT + s.name AS stream_name, + s.source_name AS source_table, + 'Snowflake Stream' AS gap_type, + 'Replace with producer cutover to ClickHouse (scripts/03_cutover.sh)' AS remediation +FROM TABLE(NYC_TAXI_DB.INFORMATION_SCHEMA.STREAMS_IN_SCHEMA('RAW')) s; +" 2>/dev/null || echo "(requires ACCOUNTADMIN — check manually: SHOW STREAMS IN SCHEMA NYC_TAXI_DB.RAW)") + +echo "${STREAM_GAPS}" >> "${OUTPUT_FILE}" +echo "" >> "${OUTPUT_FILE}" + +cat >> "${OUTPUT_FILE}" <<'MD' + +### Gap 4: MERGE INTO Patterns + +MERGE INTO does not exist in ClickHouse. dbt-clickhouse uses `delete_insert` incremental strategy +as the equivalent. Snowflake tasks using MERGE INTO must be rewritten. + +Affected objects (based on known lab setup): +- `HOURLY_AGG_TASK` — uses MERGE INTO AGG_HOURLY_ZONE_TRIPS → rewrite as dbt incremental model +- `CDC_CONSUME_TASK` — uses MERGE INTO from stream → retired after producer cutover; live writes go directly to ClickHouse + +See: https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Gap 4. + +### Gap 5: Snowflake Tasks + +Tasks are Snowflake's scheduled execution mechanism. ClickHouse has no equivalent. + +Replacements: +- Tasks running dbt models → run dbt on your own schedule (cron, Airflow, dbt Cloud) +- Tasks consuming Streams → retired after producer cutover (live writes go directly to ClickHouse) +- REFRESHABLE MATERIALIZED VIEW in ClickHouse replaces Snowflake scheduled tasks that recalculate aggregates + +### Gap 6: Date Function Differences + +Minor syntax differences — all mechanical substitutions. +See the full translation table at https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Section 2, Gap 6. + +Key substitutions needed in this workload: +- `DATE_TRUNC('hour', pickup_at)` → `toStartOfHour(pickup_at)` +- `DATEADD('day', -7, CURRENT_DATE)` → `today() - 7` or `addDays(today(), -7)` +- `DATEDIFF('minute', pickup_at, dropoff_at)` → `dateDiff('minute', pickup_at, dropoff_at)` + +--- + +## Summary + +| Gap | Objects Affected | Priority | +|-----|-----------------|----------| +| QUALIFY clause | Q3 query | High — breaks at parse time | +| VARIANT columns | TRIPS_RAW.TRIP_METADATA | High — all JSON queries affected | +| Snowflake Streams | TRIPS_CDC_STREAM | High — retired after producer cutover to ClickHouse | +| MERGE INTO | HOURLY_AGG_TASK, CDC_CONSUME_TASK | High — tasks must be rewritten | +| Snowflake Tasks | CDC_CONSUME_TASK, HOURLY_AGG_TASK | Medium — no ClickHouse equivalent | +| Date functions | Q1, Q3, Q4 queries | Low — mechanical substitutions | + +MD + +ok "Section 4 written" + +# ── Done ────────────────────────────────────────────────────── +echo "" +echo -e "${GREEN}${BOLD}════════════════════════════════════════════${RESET}" +echo -e "${GREEN}${BOLD} Profile report complete.${RESET}" +echo "" +echo " Output: ${OUTPUT_FILE}" +echo "" +echo " Next steps:" +echo " 1. Review profile_report.md" +echo " 2. Work through the worksheets:" +echo " worksheets/01_mergetree_engine_selection.md" +echo " worksheets/02_sort_key_design.md" +echo " worksheets/03_schema_translation.md" +echo " worksheets/04_migration_wave_plan.md" +echo " 3. Fill in migration-plan.md" +echo -e "${GREEN}${BOLD}════════════════════════════════════════════${RESET}" diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/02_query_history.sql b/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/02_query_history.sql new file mode 100644 index 0000000..c625d8d --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/02_query_history.sql @@ -0,0 +1,128 @@ +-- ============================================================ +-- 02_query_history.sql — Manual query history fallback +-- +-- Run this in the Snowflake UI if 01_profile_snowflake.sh cannot +-- access ACCOUNT_USAGE (requires ACCOUNTADMIN + 1-3hr lag). +-- +-- Usage: +-- 1. Open Snowflake UI → Worksheets +-- 2. Set role to ACCOUNTADMIN, warehouse to ANALYTICS_WH +-- 3. Run each query block below +-- 4. Paste results into profile_report.md Section 2 +-- ============================================================ + +-- ── Query 1: Top queries by total elapsed time ─────────────── +-- Use to identify the most expensive query patterns in the last 7 days. +-- These are the queries whose filter columns should drive your ORDER BY design. + +SELECT + query_type, + LEFT(query_text, 300) AS query_preview, + execution_status, + COUNT(*) AS executions, + ROUND(AVG(total_elapsed_time)) AS avg_ms, + ROUND(MAX(total_elapsed_time)) AS max_ms, + ROUND(MIN(total_elapsed_time)) AS min_ms, + ROUND(SUM(bytes_scanned) / 1024 / 1024 / 1024, 2) AS total_gb_scanned, + ROUND(AVG(rows_produced)) AS avg_rows_returned +FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY +WHERE database_name = 'NYC_TAXI_DB' + AND start_time >= DATEADD(day, -7, CURRENT_TIMESTAMP()) + AND execution_status = 'SUCCESS' + AND query_type NOT IN ('SHOW', 'DESCRIBE', 'USE', 'SET', 'COMMIT', 'BEGIN_TRANSACTION') + AND total_elapsed_time > 100 -- filter out sub-100ms metadata queries +GROUP BY 1, 2, 3 +ORDER BY SUM(total_elapsed_time) DESC +LIMIT 20; + + +-- ── Query 2: Filter column frequency ───────────────────────── +-- Identify which columns appear most often in WHERE clauses. +-- The most-filtered columns should be first in ORDER BY. +-- Note: This is an approximation — ACCOUNT_USAGE stores query text, not parsed ASTs. + +SELECT + CASE + WHEN LOWER(query_text) LIKE '%pickup_at%' THEN 'pickup_at' + WHEN LOWER(query_text) LIKE '%pickup_datetime%' THEN 'pickup_datetime' + WHEN LOWER(query_text) LIKE '%pickup_location_id%' THEN 'pickup_location_id' + WHEN LOWER(query_text) LIKE '%dropoff_location_id%' THEN 'dropoff_location_id' + WHEN LOWER(query_text) LIKE '%vendor_id%' THEN 'vendor_id' + WHEN LOWER(query_text) LIKE '%payment_type%' THEN 'payment_type' + WHEN LOWER(query_text) LIKE '%trip_id%' THEN 'trip_id' + ELSE 'other' + END AS filter_column, + COUNT(*) AS query_count +FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY +WHERE database_name = 'NYC_TAXI_DB' + AND start_time >= DATEADD(day, -7, CURRENT_TIMESTAMP()) + AND execution_status = 'SUCCESS' + AND query_type = 'SELECT' + AND LOWER(query_text) LIKE '%where%' +GROUP BY 1 +ORDER BY 2 DESC; + + +-- ── Query 3: Queries using Snowflake-specific constructs ────── +-- Find queries that use constructs requiring ClickHouse translation. + +SELECT + CASE + WHEN LOWER(query_text) LIKE '%qualify%' THEN 'QUALIFY' + WHEN LOWER(query_text) LIKE '%lateral flatten%' THEN 'LATERAL FLATTEN' + WHEN LOWER(query_text) LIKE '%merge into%' THEN 'MERGE INTO' + WHEN LOWER(query_text) LIKE '%::%' THEN 'VARIANT colon-path' + WHEN LOWER(query_text) LIKE '%metadata$%' THEN 'Stream METADATA$' + ELSE 'other' + END AS construct, + COUNT(*) AS query_count, + ROUND(AVG(total_elapsed_time)) AS avg_ms +FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY +WHERE database_name = 'NYC_TAXI_DB' + AND start_time >= DATEADD(day, -7, CURRENT_TIMESTAMP()) + AND execution_status = 'SUCCESS' + AND ( + LOWER(query_text) LIKE '%qualify%' + OR LOWER(query_text) LIKE '%lateral flatten%' + OR LOWER(query_text) LIKE '%merge into%' + OR query_text LIKE '%::%' + OR LOWER(query_text) LIKE '%metadata$%' + ) +GROUP BY 1 +ORDER BY 2 DESC; + + +-- ── Query 4: Warehouse utilization ─────────────────────────── +-- Understand which warehouses are doing what work. +-- Relevant for Part 3 cost comparison. + +SELECT + warehouse_name, + query_type, + COUNT(*) AS query_count, + ROUND(SUM(total_elapsed_time) / 1000 / 60, 1) AS total_minutes, + ROUND(AVG(total_elapsed_time)) AS avg_ms +FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY +WHERE database_name = 'NYC_TAXI_DB' + AND start_time >= DATEADD(day, -7, CURRENT_TIMESTAMP()) + AND execution_status = 'SUCCESS' +GROUP BY 1, 2 +ORDER BY SUM(total_elapsed_time) DESC; + + +-- ── Query 5: Table access frequency ────────────────────────── +-- Which tables are accessed most often? Guides migration wave priority. + +SELECT + ao.object_name AS table_name, + ao.object_schema AS schema_name, + COUNT(DISTINCT ah.query_id) AS distinct_queries, + ROUND(AVG(qh.total_elapsed_time)) AS avg_query_ms +FROM SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY ah +JOIN SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY qh ON ah.query_id = qh.query_id, +LATERAL FLATTEN(input => ah.base_objects_accessed) ao +WHERE qh.database_name = 'NYC_TAXI_DB' + AND qh.start_time >= DATEADD(day, -7, CURRENT_TIMESTAMP()) + AND ao.value:objectDomain::STRING = 'Table' +GROUP BY 1, 2 +ORDER BY 3 DESC; diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md new file mode 100644 index 0000000..6dbbb09 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md @@ -0,0 +1,120 @@ +# Worksheet 1: MergeTree Engine Selection + +**Time estimate:** 15–20 minutes +**Reference:** [MergeTree Engines](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/mergetree-engines) + +--- + +## Concept + +In Snowflake, you create a table and Snowflake decides how to store it. In ClickHouse, you choose a storage engine — and this choice determines correctness, not just performance. + +The three engines you need for this lab: + +**MergeTree** — The base engine. Data is stored in sorted columnar files. No deduplication. Use this when the table is insert-only or when your pipeline manages updates externally (e.g., full reload on every dbt run). + +**ReplacingMergeTree(version_col)** — Extends MergeTree with background deduplication. When rows with the same `ORDER BY` key exist in multiple parts, only the row with the highest `version_col` value is kept after a merge. Use this when rows can be updated and you have a column that increases monotonically on every update (e.g., `updated_at` timestamp). + +> **Critical gotcha:** Deduplication is asynchronous. Until ClickHouse runs a background merge, both the old and new version of a row coexist. Always use `SELECT ... FINAL` on ReplacingMergeTree tables to force deduplication at query time. + +**AggregatingMergeTree** — Extends MergeTree with partial aggregate state merging. Use this when the table stores combinable aggregate states (e.g., HyperLogLog sketches, quantile digests) and you need background aggregation. Not needed for the NYC Taxi lab — the aggregation tables are rebuilt by dbt, not accumulated. + +### Decision Tree + +``` +Does the table receive UPDATE or DELETE operations? +│ +├── No (insert-only) +│ └─► MergeTree() +│ +└── Yes + ├── Is there a timestamp/version column that increases on every update? + │ ├── Yes → ReplacingMergeTree(version_col) + │ └── No (e.g., full-reload dimension tables) + │ └─► MergeTree() — dbt handles upsert via atomic table swap (full rebuild) + │ + └── Does the table store partial aggregate states (AggregateFunction types)? + └── Yes → AggregatingMergeTree() +``` + +### Staging Models Are Always Views + +In dbt + ClickHouse, staging models should be materialized as **views**, not tables. Views have zero storage cost and are always fresh — they are just saved SQL, not physical objects. + +The engine selection exercise below covers **analytics-layer dbt models** (fact tables, aggregates, dimensions) and the ClickHouse Materialized View. It does not include `trips_raw` or staging models: + +- **`trips_raw`** is a base ClickHouse table created directly by the migration script (`scripts/02_migrate_trips.py`) — not a dbt model. It uses `ReplacingMergeTree(_synced_at)` because the migration script may retry a batch and re-insert the same `trip_id`. After cutover, the live producer can also retry on transient failures; `_synced_at DateTime DEFAULT now()` ensures the most recent write wins. +- **`stg_trips`** is a dbt **view** on top of `trips_raw`. It applies `SELECT ... FROM trips_raw FINAL` to resolve any unmerged duplicates before the data reaches downstream analytics models. Deduplication responsibility belongs here — not in a ReplacingMergeTree staging table. + +--- + +## Exercise: Engine Selection for NYC Taxi Tables + +For each table below: +1. Determine the update pattern from the description +2. Identify the version column (if any) +3. Select the correct engine +4. Write a one-sentence reasoning + +Fill in the `?` cells: + +| Table | Update Pattern | Version Column | Recommended Engine | Reasoning | +|-------|----------------|----------------|--------------------|-----------| +| `fact_trips` | Trips can be corrected after the fact (fare adjustments, status changes) — same `trip_id` re-inserted with new values | ? | ? | ? | +| `agg_hourly_zone_trips` | dbt recalculates the last 2 hours on each run and re-inserts results | ? | ? | ? | +| `dim_taxi_zones` | dbt runs a full rebuild on each run (atomic table swap (full rebuild)) | — | ? | ? | +| `dim_payment_type` | dbt runs a full rebuild on each run (atomic table swap (full rebuild)) | — | ? | ? | +| `dim_vendor` | dbt runs a full rebuild on each run (atomic table swap (full rebuild)) | — | ? | ? | +| `mv_hourly_revenue` | ClickHouse REFRESHABLE MV recalculates on a schedule | — | ? | ? | + +**Hints:** +- `fact_trips`: What does "same trip_id re-inserted with new values" mean for deduplication? +- `agg_hourly_zone_trips`: What happens when dbt inserts a new row for `(2024-01-01 14:00, zone_id=1)` that already exists from the last run? +- `dim_*` tables: dbt's `table` materialization in dbt-clickhouse does atomic table swap (full rebuild), replacing all rows on every run. Does this require deduplication? +- `mv_hourly_revenue`: A REFRESHABLE MV replaces its entire result set on each refresh. What engine fits a table that is always fully replaced? + +--- + +## Reflection Questions + +Before checking the answer key, answer these questions in your own words: + +1. Why can't you use `MergeTree()` for `fact_trips` if trips can be updated? + +2. What happens if you query `fact_trips` without `FINAL` on a `ReplacingMergeTree` table? + +3. Why is `AggregatingMergeTree` not needed for `agg_hourly_zone_trips` even though it's an aggregation table? + +--- + +
+▶ Answer Key — try the exercises first before expanding + +| Table | Update Pattern | Version Column | Recommended Engine | Reasoning | +|-------|----------------|----------------|--------------------|-----------| +| `fact_trips` | Upsert (trip corrections) | `updated_at` | `ReplacingMergeTree(updated_at)` | Same trip_id can arrive twice with different fare/status values. RMT keeps the row with the highest `updated_at`. Always query with `FINAL`. | +| `agg_hourly_zone_trips` | Upsert (rolling 2-hr recalculation) | `updated_at` | `ReplacingMergeTree(updated_at)` | dbt recalculates the last 2 hours and re-inserts. Without RMT, old and new aggregates accumulate and double-count. `updated_at` set to `now()` on each dbt run ensures the latest values win. | +| `dim_taxi_zones` | Full reload | — | `MergeTree()` | dbt's `table` materialization does atomic table swap (full rebuild), so there is never more than one generation of rows. No deduplication needed. | +| `dim_payment_type` | Full reload | — | `MergeTree()` | Same as `dim_taxi_zones`. | +| `dim_vendor` | Full reload | — | `MergeTree()` | Same as `dim_taxi_zones`. | +| `mv_hourly_revenue` | Full replacement on each REFRESH | — | `MergeTree()` | REFRESHABLE MVs replace their entire result set atomically. No duplicates can accumulate. | + +**Reflection answers:** + +1. With `MergeTree`, every version of a trip row is kept permanently. If trip `T001` is inserted twice with different fare amounts, both rows appear in `SELECT * ... WHERE trip_id = 'T001'`, and `SUM(fare_amount)` doubles. ReplacingMergeTree deduplicates on merge so only the latest version is retained. + +2. Without `FINAL`, ClickHouse may return multiple versions of the same row — both the original insert and any updates — because background merges are asynchronous. Aggregates (SUM, COUNT) over the full table will over-count until the next merge runs. `FINAL` forces synchronous deduplication at read time. + +3. `AggregatingMergeTree` is for tables where rows store *partial aggregate states* (binary `AggregateFunction` columns) that need to be merged across rows. `agg_hourly_zone_trips` stores final values (INT, FLOAT) that must be deduplicated via upsert — the latest row wins, it is not combined with prior rows. `ReplacingMergeTree` is correct here; `AggregatingMergeTree` would require rewriting all queries to use `-Merge` suffix combiners. + +
+ +--- + +## Transfer to migration-plan.md + +Once you have filled in this worksheet, copy your engine decisions to Section 3 of `migration-plan.md` and check off: + +``` +- [ ] Engine selection: completed +``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md new file mode 100644 index 0000000..c8166cc --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md @@ -0,0 +1,157 @@ +# Worksheet 2: Sort Key Design (ORDER BY) + +**Time estimate:** 20–25 minutes +**Reference:** [MergeTree Engines](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/mergetree-engines) — ORDER BY Design section + +--- + +## Concept + +ClickHouse's `ORDER BY` clause is not decorative. It defines the **primary index** — a sparse, block-level index that allows ClickHouse to skip irrelevant blocks of data when evaluating `WHERE` clauses. It also determines the physical sort order of data within parts, which affects compression. + +**The wrong ORDER BY = slow queries + wasted storage.** A UUID-first ORDER BY means no block skipping for any analytical query (UUIDs are random; there is no sortable prefix). A date-first ORDER BY means queries filtering on date skip most of the table. + +### Three rules for ORDER BY design + +**Rule 1: Derive from query filters, not from the source schema.** +Look at the `WHERE`, `GROUP BY`, and `JOIN` columns across your most frequent queries. The most-filtered columns should probably appear in `ORDER BY` (Provided it is not too high cardinality). The source table's primary key (if any) is usually irrelevant. + +**Rule 2: Low cardinality first, high cardinality last.** +ClickHouse's primary index has one entry per ~8192 rows (a *granule*). Low-cardinality columns (e.g., `toStartOfMonth(date)` = ~48 distinct values over 4 years) cluster many rows together — the index can skip entire granules. High-cardinality columns (e.g., `trip_id` = 50M distinct values) are unique per row — putting them first means the index cannot skip anything. + +**Rule 3: For ReplacingMergeTree, end with the unique row identifier.** +The deduplication key is the full `ORDER BY` tuple. If `trip_id` is missing from `ORDER BY`, two different trips with the same `pickup_at` and no further columns would be treated as duplicates. Put `trip_id` last to ensure uniqueness without hurting index performance. + +--- + +## Exercise: Query Workload Analysis + +Before designing sort keys, analyze what columns the queries actually filter on. + +The NYC Taxi lab has 7 representative queries. For each, identify the filter columns (WHERE, GROUP BY/ORDER BY on fact tables, JOIN keys): + +| Query | Description | Filter / Group Columns | Most selective filter | +|-------|-------------|----------------------|----------------------| +| Q1 | Hourly revenue by borough (DATE_TRUNC, DATEADD) | `pickup_at` (date range), `borough` (via dim join) | ? | +| Q2 | Rolling 7-day avg distance (window function) | `pickup_at::DATE` (GROUP BY — no WHERE filter) | ? | +| Q3 | Top 10 trips per zone (QUALIFY) | `pickup_at` (single day: yesterday), `pickup_borough` (PARTITION BY in window) | ? | +| Q4 | Driver ratings by zone (LATERAL FLATTEN / JSON) | JSON metadata fields (`driver` not null, `driver.rating` not null) | ? | +| Q5 | Surge pricing analysis (VARIANT colon-path) | `pickup_at` (date range) | ? | +| Q6 | Hourly aggregation (MERGE equivalent) | `hour_bucket`, `zone_id` | ? | +| Q7 | CDC stream lag measurement | `trip_id` (point lookup) | ? | + +Fill in the "Most selective filter" column: which column, when filtered, eliminates the most rows? + +--- + +## Exercise: Cardinality Estimation + +For each candidate ORDER BY column, estimate its cardinality over the 4-year, 50M-row dataset: + +| Column | Estimated Distinct Values | Cardinality Class | +|--------|--------------------------|------------------| +| `toStartOfMonth(pickup_at)` | ~48 (4 years × 12 months) | Very low | +| `toStartOfDay(pickup_at)` | ~1,460 (4 years × 365 days) | Low | +| `pickup_at` (DateTime64) | ~? | Medium-high | +| `pickup_location_id` | 265 (TLC zones) | Very low | +| `zone_id` | 265 | Very low | +| `hour_bucket` (by hour) | ~35,040 (4 years × 24 × 365) | Medium | +| `borough` | 6 | Extremely low | +| `payment_type` | 6 | Extremely low | +| `vendor_id` | 3 | Extremely low | +| `trip_id` (String UUID) | 50,000,000 | Extremely high | + +Fill in the `pickup_at` estimated cardinality. (Hint: the trip producer inserts ~60 trips/minute. What is 60 trips/minute × 4 years in seconds?) + +--- + +## Exercise: Sort Key Design + +Using your query workload analysis and cardinality estimates, propose an `ORDER BY` for each table: + +**Rules reminder:** +- Low cardinality first → most block-skipping +- Columns that appear in WHERE/GROUP BY of multiple queries → include them +- For ReplacingMergeTree tables → end with the unique row identifier +- Don't include columns that are never filtered on + +| Table | Engine | Proposed ORDER BY | Reasoning | +|-------|--------|-------------------|-----------| +| `trips_raw` | ReplacingMergeTree(_synced_at) | `(?, ?)` | *Python migration script loads data; time-range queries filter on pickup_at; trip_id must be in ORDER BY as the RMT dedup key* | +| `fact_trips` | ReplacingMergeTree(updated_at) | `(?, ?, ?)` | *Q1-Q7 all filter on pickup_at; trip_id needed for RMT uniqueness* | +| `agg_hourly_zone_trips` | ReplacingMergeTree(updated_at) | `(?, ?)` | *Aggregation queries filter on hour_bucket and zone_id* | +| `dim_taxi_zones` | MergeTree | `(location_id)` | *(filled as example — dim tables are small, lookup by PK)* | + +**Hints for `trips_raw`:** +- CDC lookups (`WHERE trip_id = ?`) need trip_id in ORDER BY for efficient point lookups +- Analytical range queries (`WHERE pickup_at BETWEEN ...`) benefit from pickup_at + +**Hints for `fact_trips`:** +- All 7 analytical queries filter on `pickup_at` in some form +- `toStartOfMonth(pickup_at)` as the first column groups an entire month's data into adjacent blocks — this enables coarse-grained block skipping even without PARTITION BY +- `trip_id` must be last (RMT dedup key) + +**Hints for `agg_hourly_zone_trips`:** +- Q6 groups by `hour_bucket` and `zone_id` — these are the only filter columns +- Which has lower cardinality? + +--- + +## Reflection Questions + +1. Why does `toStartOfMonth(pickup_at)` appear as the first column in `fact_trips` ORDER BY, rather than just `pickup_at`? + +2. If Q7 does a point lookup by `trip_id` on `trips_raw`, should `trip_id` be first in `trips_raw`'s ORDER BY? Why or why not? + +3. `dim_taxi_zones` has only 265 rows. Does the ORDER BY choice matter for this table? Explain. + +--- + +
+▶ Answer Key — try the exercises first before expanding + +### Query Workload Analysis + +| Query | Most selective filter | +|-------|-----------------------| +| Q1 | `pickup_at` (date range eliminates 95%+ of rows before borough join) | +| Q2 | No WHERE filter — full table scan grouped by `pickup_at::DATE` | +| Q3 | `pickup_at` (single day — eliminates ~99.7% of 4-year dataset) | +| Q4 | JSON metadata presence check — not a sortable/skippable column | +| Q5 | `pickup_at` (date range) | +| Q6 | `hour_bucket` (specific time bucket) | +| Q7 | `trip_id` (UUID point lookup — unique per row) | + +### Cardinality: pickup_at + +60 trips/min × 60 min/hr × 24 hr/day × 365 days/yr × 4 years = ~126M possible minute-slots across 4 years. At second precision, even more. The 50M trips span ~2.1M distinct seconds → very high cardinality. Using `pickup_at` raw as the first ORDER BY column gives poor block skipping. `toStartOfMonth(pickup_at)` gives ~48 distinct values — much more useful as a first sort key. + +### Sort Key Answers + +| Table | Proposed ORDER BY | Reasoning | +|-------|-------------------|-----------| +| `trips_raw` | `(pickup_at, trip_id)` | Time-range analytical queries benefit from pickup_at first. trip_id for CDC point lookups (RMT uses it as dedup key if needed). | +| `fact_trips` | `(toStartOfMonth(pickup_at), pickup_at, trip_id)` | Month prefix clusters entire months together (block-level skipping). pickup_at narrows within month. trip_id ensures RMT uniqueness. Q1-Q7 all filter on pickup_at in some form — no query filters on trip_id alone on the fact table. **Production alternative:** `PARTITION BY toStartOfMonth(pickup_at)` + `ORDER BY (pickup_at, trip_id)` is equally valid — partitioning handles month-level pruning at the filesystem level, freeing the ORDER BY to focus purely on within-partition range scans. The lab uses the function-in-ORDER BY approach to keep the DDL simple, but both are production-grade patterns. | +| `agg_hourly_zone_trips` | `(hour_bucket, zone_id)` | Both columns appear in Q6. hour_bucket has medium cardinality (~35K); zone_id has low cardinality (265). In query patterns, `WHERE hour_bucket >= X` narrows more than `WHERE zone_id = Y`, but zone_id is a secondary filter. Both should be in ORDER BY. | + +### Reflection Answers + +1. `toStartOfMonth(pickup_at)` as the first column groups all trips in a calendar month into adjacent storage blocks. ClickHouse's sparse index can skip all blocks outside the requested date range at month granularity, before `pickup_at` narrows further within the month. If `pickup_at` were first alone, every granule would cover a mix of timestamps with no coarse structure — the index would still work but less efficiently for month-level range scans. + + > **Production note:** An equally valid design is `PARTITION BY toStartOfMonth(pickup_at)` + `ORDER BY (pickup_at, trip_id)`. `PARTITION BY` handles month-level pruning at the filesystem level (entire partition directories are skipped before any index is consulted), while `ORDER BY` focuses purely on within-partition range scans. The trade-off: partitioning creates more files and can hurt performance if partitions are too small (e.g., daily partitions on a sparse dataset). For 50M rows across 4 years (~1M rows/month), monthly partitioning is a safe choice. The lab keeps `toStartOfMonth` in ORDER BY to avoid introducing `PARTITION BY` as a separate concept, but SAs should know both patterns. + +2. No. `trip_id` has extremely high cardinality (UUID = 50M distinct values). Putting it first means every block contains exactly one `trip_id` value range — the index is useless for anything except exact `trip_id` lookups. Since the vast majority of `trips_raw` queries are analytical (time-range scans), not CDC point lookups, `pickup_at` first is more valuable. The CDC lookup performance penalty for `trip_id` second is acceptable because CDC lookups are rare point-in-time operations, not the primary analytical workload. + +3. `dim_taxi_zones` has 265 rows — one granule (8192 rows per granule). ClickHouse will read the entire table in a single granule regardless of ORDER BY. The choice does not matter for performance. `(location_id)` is still good practice for clarity and for join performance when ClickHouse uses a hash join and needs to look up rows by location_id. + +
+ +--- + +## Transfer to migration-plan.md + +Copy your ORDER BY decisions to Section 4 of `migration-plan.md` and check off: + +``` +- [ ] Sort key design: completed +``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md new file mode 100644 index 0000000..e30edcd --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md @@ -0,0 +1,204 @@ +# Worksheet 3: Schema Translation + +**Time estimate:** 20–25 minutes +**Reference:** [Snowflake vs ClickHouse](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse) — Section 2 (SQL Dialect Gaps) + +--- + +## Concept + +Type mapping and function translation are the most mechanical part of migration, but also the most error-prone if done carelessly. Snowflake and ClickHouse have different type systems with different semantics, and using the wrong type can cause silent precision loss, excessive storage, or broken query logic. + +**Key principles:** + +1. **Be explicit about precision.** Snowflake's `TIMESTAMP_NTZ` has nanosecond precision. ClickHouse's `DateTime` has only second precision — do not use it for version columns. Use `DateTime64(3, 'UTC')` for millisecond precision (matching most real-world requirements) or `DateTime64(9, 'UTC')` for nanoseconds. **This matters for correctness:** if a `ReplacingMergeTree` version column has only second precision, two updates arriving within the same second are non-deterministic — ClickHouse cannot determine which is newer. + +2. **Use the smallest correct integer type.** Snowflake's `INTEGER` is `NUMBER(38, 0)` — 38-digit fixed precision stored as a 128-bit value. ClickHouse has fixed-width integers: `Int8`, `Int16`, `Int32`, `Int64`, `UInt8`, `UInt16`, `UInt32`, `UInt64`. Choosing `UInt8` for `vendor_id` (values 1–3) saves 7 bytes per row vs `Int64`. At 50M rows, that is 350MB. + +3. **VARIANT → String.** ClickHouse has a native `JSON` type (available in v25.3+ as production-stable), but it is designed for truly dynamic schemas where the field names and structure are unknown at table creation time. For `trip_metadata` in this lab, the structure is known (`driver.rating`, `app.surge_multiplier`, etc.) — the better approach is to pre-flatten into typed columns during migration, or store as `String` and use `JSONExtract*` at query time. Use the `JSON` type when you genuinely cannot predict the schema: e.g., ingesting arbitrary customer event payloads where every event has different fields. + +4. **Float precision.** Snowflake's `FLOAT` maps to `Float64` in ClickHouse. For monetary amounts where exact decimal arithmetic is required, use `Decimal(18, 2)` — but for this lab, `Float64` is sufficient to match the source. + +5. **LowCardinality() — ClickHouse-only optimization.** Wrapping a type in `LowCardinality(String)` (or `LowCardinality(UInt8)`, etc.) tells ClickHouse to use a dictionary encoding for that column — values are stored as integer references to a dictionary rather than repeated strings. This typically gives 2–5x compression improvement and faster GROUP BY on string columns with fewer than ~10,000 distinct values. Snowflake has no equivalent; it handles this automatically. Good candidates in this lab: `pickup_borough` (6 values), `payment_type` (6 values), `vehicle_type`, `vendor_name`. + +--- + +## Exercise 1: Type Mapping for TRIPS_RAW + +Map each column from `NYC_TAXI_DB.RAW.TRIPS_RAW` to its ClickHouse type. Fill in the `?` cells: + +| Snowflake Column | Snowflake Type | ClickHouse Type | Notes | +|------------------|---------------|-----------------|-------| +| `TRIP_ID` | `VARCHAR(36)` | `String` | *(ClickHouse has a native [`UUID`](https://clickhouse.com/docs/sql-reference/data-types/uuid) type, but `String` is idiomatic when migrating from VARCHAR(36): it requires no casting, supports all string functions, and avoids UUID parsing overhead on insert)* | +| `PICKUP_DATETIME` | `TIMESTAMP_NTZ(9)` | ? | *Hint: what precision does DateTime64 support? What timezone?* | +| `DROPOFF_DATETIME` | `TIMESTAMP_NTZ(9)` | ? | *Same as PICKUP_DATETIME* | +| `PICKUP_LOCATION_ID` | `INTEGER` | ? | *Values 1–265. What is the smallest correct unsigned integer type?* | +| `DROPOFF_LOCATION_ID` | `INTEGER` | ? | *Same as PICKUP_LOCATION_ID* | +| `VENDOR_ID` | `INTEGER` | ? | *Values 1–3. What is the smallest correct unsigned integer type?* | +| `PASSENGER_COUNT` | `INTEGER` | ? | *Values 1–6 typically. What type?* | +| `TRIP_DISTANCE` | `FLOAT` | ? | *Decimal miles — precision matters somewhat but not critically* | +| `FARE_AMOUNT` | `FLOAT` | ? | *Monetary. Float64 is acceptable for this lab.* | +| `TIP_AMOUNT` | `FLOAT` | ? | *Same as FARE_AMOUNT* | +| `TOTAL_AMOUNT` | `FLOAT` | ? | *Same as FARE_AMOUNT* | +| `PAYMENT_TYPE_ID` | `INTEGER` | ? | *Values 1–6. What type?* | +| `TRIP_METADATA` | `VARIANT` | ? | *JSON blob — the key migration challenge* | +| `CREATED_AT` | `TIMESTAMP_NTZ(9)` | ? | *Row creation timestamp* | + +--- + +## Exercise 2: Type Mapping for FACT_TRIPS + +`FACT_TRIPS` adds computed/derived columns that were added by the dbt pipeline: + +| Snowflake Column | Snowflake Type | ClickHouse Type | Notes | +|------------------|---------------|-----------------|-------| +| `TRIP_ID` | `VARCHAR(36)` | `String` | *(filled)* | +| `PICKUP_AT` | `TIMESTAMP_NTZ(9)` | ? | | +| `DROPOFF_AT` | `TIMESTAMP_NTZ(9)` | ? | | +| `PICKUP_LOCATION_ID` | `INTEGER` | ? | | +| `ZONE_ID` | `INTEGER` | ? | *Foreign key to dim_taxi_zones; same cardinality as location IDs* | +| `FARE_AMOUNT` | `FLOAT` | ? | | +| `TRIP_DISTANCE` | `FLOAT` | ? | | +| `DRIVER_RATING` | `FLOAT` | ? | *Values 1.0–5.0; can be NULL (no rating)* | +| `PAYMENT_TYPE_ID` | `INTEGER` | ? | | +| `VENDOR_ID` | `INTEGER` | ? | | +| `UPDATED_AT` | `TIMESTAMP_NTZ(9)` | ? | *Version column for ReplacingMergeTree — precision matters* | + +**Note on nullable columns:** In ClickHouse, `Nullable(Float64)` has a slight performance overhead compared to `Float64`. For `driver_rating`, which is frequently NULL, you have two options: +- `Nullable(Float32)` — explicit null semantics; slower on aggregations due to null-tracking overhead (a separate bitmask column is stored alongside the data) +- `Float32` with `nan` or `-1.0` as sentinel value — faster but less conventional + +For the lab, use `Nullable(Float32)` for correctness. + +--- + +## Exercise 3: Function Translation + +Translate each Snowflake expression to its ClickHouse equivalent. These come directly from Q1–Q7 in `01-setup-snowflake/queries/`. + +| Snowflake Expression | Query | ClickHouse Equivalent | +|----------------------|-------|----------------------| +| `DATE_TRUNC('hour', pickup_at)` | Q1 | ? | +| `DATEADD('day', -7, CURRENT_DATE)` | Q1, Q3 | ? | +| `DATEDIFF('minute', pickup_at, dropoff_at)` | Q4 | ? | +| `TRIP_METADATA:driver.rating::FLOAT` | Q4, Q5 | ? | +| `TRIP_METADATA:app.surge_multiplier::FLOAT` | Q5 | ? | +| `QUALIFY ROW_NUMBER() OVER (PARTITION BY pickup_location_id ORDER BY fare_amount DESC) <= 10` | Q3 | ? — treated as a dialect gap in this lab; requires structural rewrite | +| `MERGE INTO fact_trips t USING staging s ON t.trip_id = s.trip_id WHEN MATCHED THEN UPDATE ...` | Q6 | ? — requires structural rewrite *(Hint: think about the engine and dbt strategy you chose in Worksheets 1 & 2)* | +| `SELECT METADATA$ACTION, METADATA$ISUPDATE FROM trips_cdc_stream` | Q7 | ? — no equivalent needed; live writes go directly to ClickHouse post-cutover; what pattern gives you current state? | + +**Hints:** +- `DATE_TRUNC('hour', ...)` → look at `toStartOfHour` family in ClickHouse docs +- `DATEADD('day', -7, ...)` → ClickHouse uses `today() - 7` for dates, or `now() - INTERVAL 7 DAY` +- `DATEDIFF('minute', t1, t2)` → ClickHouse has `dateDiff` (lowercase d) +- `TRIP_METADATA:driver.rating::FLOAT` → nested path requires two-level `JSONExtractFloat` +- QUALIFY → for this lab, treated as a dialect gap; wrap in a subquery: keep the window function, add `WHERE` outside (this rewrite is portable across all SQL engines) + +--- + +## Exercise 4: Non-Obvious Translation Decisions + +For each decision below, write your reasoning (not just the answer): + +**Decision 1:** `TRIP_METADATA` is `VARIANT` in Snowflake. You have three options in ClickHouse: +- `String` — store raw JSON; use `JSONExtract*` at query time +- `Map(String, String)` — structured key-value; loses nested structures +- `Tuple(...)` — fixed schema; breaks if JSON structure varies + +Which do you choose and why? + +*Your answer:* + +--- + +**Decision 2:** `FARE_AMOUNT` is `FLOAT` in Snowflake. In ClickHouse, you could use `Float64`, `Float32`, or `Decimal(18, 2)`. The lab has 50M rows. + +Which do you choose and why? (Consider: precision needs, storage cost, and whether fare amounts in this workload require exact decimal arithmetic.) + +*Your answer:* + +--- + +**Decision 3:** `PICKUP_LOCATION_ID` is `INTEGER` in Snowflake (values 1–265). In ClickHouse, you could use `Int32`, `Int16`, or `UInt16`. + +Calculate the storage savings of using `UInt16` instead of `Int32` at 50M rows: + +*Your calculation:* + +--- + +
+▶ Answer Key — try the exercises first before expanding + +### Exercise 1: TRIPS_RAW Types + +| Column | ClickHouse Type | Reasoning | +|--------|----------------|-----------| +| `PICKUP_DATETIME` | `DateTime64(3, 'UTC')` | Millisecond precision is sufficient; `'UTC'` makes timezone explicit. Nanosecond (`9`) is overkill for trip timestamps. | +| `DROPOFF_DATETIME` | `DateTime64(3, 'UTC')` | Same. | +| `PICKUP_LOCATION_ID` | `UInt16` | Values 1–265. `UInt8` max is 255 — too small. `UInt16` max is 65535 — correct and half the size of `Int32`. | +| `DROPOFF_LOCATION_ID` | `UInt16` | Same. | +| `VENDOR_ID` | `UInt8` | Values 1–3. `UInt8` max is 255 — correct. 1 byte per row vs 4 for Int32. | +| `PASSENGER_COUNT` | `UInt8` | Values 1–6. `UInt8` correct. | +| `TRIP_DISTANCE` | `Float64` | Float distance; no exact decimal requirement. Matches Snowflake FLOAT. | +| `FARE_AMOUNT` | `Float64` | See Exercise 4 Decision 2. | +| `TIP_AMOUNT` | `Float64` | Same. | +| `TOTAL_AMOUNT` | `Float64` | Same. | +| `PAYMENT_TYPE_ID` | `UInt8` | Values 1–6. | +| `TRIP_METADATA` | `String` | See Exercise 4 Decision 1. | +| `CREATED_AT` | `DateTime64(3, 'UTC')` | Row creation timestamp — milliseconds sufficient. | + +### Exercise 2: FACT_TRIPS Types + +| Column | ClickHouse Type | +|--------|----------------| +| `PICKUP_AT` | `DateTime64(3, 'UTC')` | +| `DROPOFF_AT` | `DateTime64(3, 'UTC')` | +| `PICKUP_LOCATION_ID` | `UInt16` | +| `ZONE_ID` | `UInt16` | +| `FARE_AMOUNT` | `Float64` | +| `TRIP_DISTANCE` | `Float64` | +| `DRIVER_RATING` | `Nullable(Float32)` | +| `PAYMENT_TYPE_ID` | `UInt8` | +| `VENDOR_ID` | `UInt8` | +| `UPDATED_AT` | `DateTime64(3, 'UTC')` — must be DateTime64 not DateTime; second precision could cause two updates in the same second to be non-deterministic | + +### Exercise 3: Function Translations + +| Snowflake Expression | ClickHouse Equivalent | +|----------------------|-----------------------| +| `DATE_TRUNC('hour', pickup_at)` | `toStartOfHour(pickup_at)` | +| `DATEADD('day', -7, CURRENT_DATE)` | `today() - 7` or `addDays(today(), -7)` | +| `DATEDIFF('minute', pickup_at, dropoff_at)` | `dateDiff('minute', pickup_at, dropoff_at)` | +| `TRIP_METADATA:driver.rating::FLOAT` | `JSONExtractFloat(trip_metadata, 'driver', 'rating')` | +| `TRIP_METADATA:app.surge_multiplier::FLOAT` | `JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier')` | +| `QUALIFY ROW_NUMBER() OVER (...) <= 10` | `SELECT ... FROM (SELECT ..., ROW_NUMBER() OVER (...) AS rn FROM ...) WHERE rn <= 10` | +| `MERGE INTO ... WHEN MATCHED THEN UPDATE` | dbt `delete_insert` incremental strategy — no direct SQL equivalent | +| `SELECT METADATA$ACTION FROM stream` | No equivalent needed — live writes go directly to ClickHouse post-cutover; query the target table with `FINAL` for current deduplicated state | + +### Exercise 4: Decisions + +**Decision 1 — TRIP_METADATA → String** +Choose `String`. `Map(String, String)` loses nested structures (the rating is nested under `driver.rating`, not a flat key). `Tuple(...)` requires a fixed schema — if the JSON structure varies across trips, the Tuple definition fails or requires frequent schema migrations. `String` preserves the raw JSON exactly, and `JSONExtract*` handles any path at query time. The trade-off is that extraction is slower than native column access, but for VARIANT columns that are queried infrequently (only in Q4, Q5), this is acceptable. + +**Decision 2 — FARE_AMOUNT → Float64** +Use `Float64`. `Float32` has ~7 significant digits of precision — a $10,000.00 fare would be stored as $9,999.999... (rounding at 7 digits), which is wrong for financial reporting. `Decimal(18, 2)` gives exact precision but is slower on aggregations. For a lab environment where we're demonstrating analytics performance, `Float64` (15–16 significant digits) is the pragmatic choice. In a production financial system, `Decimal(18, 2)` would be correct. + +**Decision 3 — UInt16 vs Int32 storage savings** +- `Int32`: 4 bytes per row × 50,000,000 rows = 200,000,000 bytes = ~190 MB +- `UInt16`: 2 bytes per row × 50,000,000 rows = 100,000,000 bytes = ~95 MB +- Savings: ~95 MB per column before compression +- ClickHouse compression (LZ4 default) compresses sorted integer columns at ~5-10x for low-cardinality values → actual storage might be 10-20 MB per column regardless +- But: two `UInt16` columns (`pickup_location_id` + `dropoff_location_id`) on 50M rows saves ~190 MB raw storage and improves compression since small integers compress better + +
+ +--- + +## Transfer to migration-plan.md + +Copy your type decisions and any non-obvious translation notes to Section 5 of `migration-plan.md` and check off: + +``` +- [ ] Schema translation: completed +``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md new file mode 100644 index 0000000..b67083d --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md @@ -0,0 +1,183 @@ +# Worksheet 4: Migration Wave Plan + +**Time estimate:** 15–20 minutes +**Reference:** [Module 03 — Provision and migrate](https://workshop.demohouse.cloud/docs/snowflake-migration/learner/03-provision-and-migrate) → [Module 04 — Rebuild the dbt pipeline](https://workshop.demohouse.cloud/docs/snowflake-migration/learner/04-rebuild-dbt) → [Module 05 — Benchmark and cutover](https://workshop.demohouse.cloud/docs/snowflake-migration/learner/05-benchmark-and-cutover) for execution order + +--- + +## Concept + +Database objects have dependencies. A view that queries `fact_trips` cannot be created before `fact_trips` exists. A materialized view that reads from `trips_raw` cannot be populated before `trips_raw` has data. Migrating in the wrong order causes create-table failures, empty results, or incomplete data that is hard to diagnose. + +**The solution: wave planning.** Organize every object into a numbered wave where each wave contains only objects whose dependencies are satisfied by prior waves. + +**Complexity grading** helps prioritize migration effort: +- **Grade A** — trivial: standard table, no special logic, straightforward type mapping +- **Grade B** — medium: dbt incremental model with ClickHouse-specific engine; requires testing +- **Grade C** — complex: requires SQL rewrite (QUALIFY, MERGE INTO, VARIANT); test carefully +- **Grade D** — requires redesign: Snowflake-specific feature with no direct equivalent (Streams → producer cutover to ClickHouse, Tasks → Refreshable MVs or scheduled dbt runs) + +--- + +## Exercise 1: Dependency DAG + +Map the dependencies between all objects in the NYC Taxi workload. Draw a line from each object to the objects it depends on. + +``` +Objects to map: + Source (Snowflake): + - TRIPS_RAW (raw table) + - TRIPS_CDC_STREAM (stream on TRIPS_RAW) + - CDC_CONSUME_TASK (task reading TRIPS_CDC_STREAM) + - HOURLY_AGG_TASK (task writing AGG_HOURLY_ZONE_TRIPS) + + ClickHouse targets (via dbt or Python migration script): + - trips_raw (ClickHouse base table) + - stg_trips (dbt view) + - stg_taxi_zones (dbt view) + - int_trips_enriched (dbt ephemeral CTE) + - fact_trips (dbt incremental) + - agg_hourly_zone_trips (dbt incremental) + - dim_taxi_zones (dbt table) + - dim_payment_type (dbt table) + - dim_vendor (dbt table) + - taxi_zones_dict (ClickHouse dictionary) + - mv_hourly_revenue (ClickHouse Refreshable MV) +``` + +Fill in the dependency table: + +| Object | Depends On | Notes | +|--------|------------|-------| +| `trips_raw` (ClickHouse) | *(nothing — first object created)* | Schema created by dbt before the Python migration script loads data | +| `stg_trips` | ? | | +| `stg_taxi_zones` | ? | | +| `int_trips_enriched` | ? | | +| `fact_trips` | ? | | +| `agg_hourly_zone_trips` | ? | | +| `dim_taxi_zones` | ? | | +| `dim_payment_type` | ? | *Source data is static reference data — does it depend on trips_raw?* | +| `dim_vendor` | ? | | +| `taxi_zones_dict` | ? | *Dictionary reads from dim_taxi_zones at creation time* | +| `mv_hourly_revenue` | ? | *Refreshable MV queries fact_trips on each refresh* | + +--- + +## Exercise 2: Migration Wave Assignment + +Using the dependency map above, assign each object to a migration wave. Objects in the same wave can be created/loaded in parallel. Objects in later waves must wait for earlier waves to complete. + +Fill in the wave assignment: + +| Wave | Objects | What happens | +|------|---------|--------------| +| Wave 0 | `trips_raw` (schema only), `dim_*` tables | ? | +| Wave 1 | Python bulk load (`scripts/02_migrate_trips.py`), `stg_trips`, `stg_taxi_zones` | ? | +| Wave 2 | ? | ? | +| Wave 3 | ? | ? | +| Wave 4 | ? | ? | + +**Constraint:** The Python bulk load (`scripts/02_migrate_trips.py`) must complete before Wave 2 dbt models run — `stg_trips`, `int_trips_enriched`, and `fact_trips` all read from `trips_raw`, which must have data. + +--- + +## Exercise 3: Complexity Grading + +Grade each object A/B/C/D and explain the primary migration challenge: + +| Object | Grade | Primary Challenge | +|--------|-------|------------------| +| `trips_raw` | ? | | +| `stg_trips` | ? | *Contains JSONExtract for TRIP_METADATA — straightforward but requires JSON path knowledge* | +| `stg_taxi_zones` | ? | | +| `int_trips_enriched` | ? | | +| `fact_trips` | ? | *ReplacingMergeTree, delete_insert incremental, QUALIFY rewrite in Q3* | +| `agg_hourly_zone_trips` | ? | | +| `dim_taxi_zones` | ? | | +| `dim_payment_type` | ? | | +| `dim_vendor` | ? | | +| `taxi_zones_dict` | ? | *ClickHouse-specific syntax; no Snowflake equivalent* | +| `mv_hourly_revenue` | ? | *Refreshable MV syntax is ClickHouse-specific; REFRESH EVERY syntax* | +| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | ? | *Snowflake-specific; replaced by direct producer cutover to ClickHouse in Part 3* | + +--- + +## Exercise 4: Risk Register + +For each Grade C/D object, identify: +1. What can go wrong +2. How you will verify correctness after migration + +| Object | Risk | Verification | +|--------|------|--------------| +| `fact_trips` | ? | | +| `agg_hourly_zone_trips` | ? | | +| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | ? | | + +--- + +
+▶ Answer Key — try the exercises first before expanding + +### Exercise 1: Dependency Table + +| Object | Depends On | +|--------|------------| +| `trips_raw` | Nothing (created first; schema before data) | +| `stg_trips` | `trips_raw` (reads from base table) | +| `stg_taxi_zones` | raw source `taxi_zones` — seeded by `scripts/00_seed_zones.sql` into `default.taxi_zones` (distinct from the analytics `dim_taxi_zones` model) | +| `int_trips_enriched` | `stg_trips`, `stg_taxi_zones` (joins both staging views) | +| `fact_trips` | `int_trips_enriched` (dbt CTE chain) | +| `agg_hourly_zone_trips` | `int_trips_enriched` (dbt CTE chain) | +| `dim_taxi_zones` | Reference data (static CSV seed — no CH dependency) | +| `dim_payment_type` | Reference data (static — no dependency) | +| `dim_vendor` | Reference data (static — no dependency) | +| `taxi_zones_dict` | `dim_taxi_zones` (reads from it at dictionary creation time) | +| `mv_hourly_revenue` | `fact_trips` (refreshable MV queries fact_trips) | + +### Exercise 2: Migration Wave Assignment + +| Wave | Objects | What happens | +|------|---------|--------------| +| Wave 0 | `trips_raw` schema, `dim_taxi_zones`, `dim_payment_type`, `dim_vendor` | dbt creates empty tables; reference dims populated immediately (static data) | +| Wave 1 | Python bulk load (`scripts/02_migrate_trips.py`) → `trips_raw` | 50M rows loaded from Snowflake; verify row count parity with `scripts/01_verify_migration.sh` | +| Wave 2 | `stg_trips`, `stg_taxi_zones`, `int_trips_enriched` (ephemeral), `fact_trips`, `agg_hourly_zone_trips` | `dbt run` builds the analytics layer from loaded data | +| Wave 3 | `taxi_zones_dict`, `mv_hourly_revenue` | Dictionary created after `dim_taxi_zones` is populated; refreshable MV created after `fact_trips` has data | +| Wave 4 | Producer cutover (`scripts/03_cutover.sh`) | Stop Snowflake producer; run final migration delta; start ClickHouse producer writing directly to ClickHouse Cloud | + +### Exercise 3: Complexity Grading + +| Object | Grade | Challenge | +|--------|-------|-----------| +| `trips_raw` | A | ReplacingMergeTree(_synced_at); migration retries and post-cutover producer retries require idempotent inserts — same decision as Worksheet 1 | +| `stg_trips` | B | JSONExtract for TRIP_METADATA; requires JSON path knowledge and testing | +| `stg_taxi_zones` | A | Passthrough view; trivial translation | +| `int_trips_enriched` | A | Ephemeral CTE; SQL differences are handled in parent models | +| `fact_trips` | C | ReplacingMergeTree engine; `delete_insert` incremental strategy; QUALIFY treated as a dialect gap in Q3 (subquery rewrite); must use FINAL in queries | +| `agg_hourly_zone_trips` | B | ReplacingMergeTree; rolling window recalculation logic; `delete_insert` strategy with 2-hr window | +| `dim_taxi_zones` | A | Static reference; full reload; trivial | +| `dim_payment_type` | A | Static reference; trivial | +| `dim_vendor` | A | Static reference; trivial | +| `taxi_zones_dict` | B | ClickHouse-specific `CREATE DICTIONARY` syntax; `HASHED()` layout; dictGet() at query time | +| `mv_hourly_revenue` | B | `REFRESH EVERY 180 SECOND` syntax; must verify it replaces data atomically | +| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | D | Snowflake Streams and Tasks have no direct ClickHouse equivalent; replaced by direct producer cutover in Part 3 — the live trip producer writes to ClickHouse Cloud after cutover | + +### Exercise 4: Risk Register + +| Object | Risk | Verification | +|--------|------|--------------| +| `fact_trips` | Queries without FINAL over-count rows during merge lag period; `delete_insert` partition range must match ORDER BY prefix to avoid deleting non-target rows | `SELECT COUNT(*) FROM fact_trips FINAL` matches Snowflake `COUNT(*)` within CDC lag tolerance; run `dbt test` and compare Q3 output between systems | +| `agg_hourly_zone_trips` | Rolling 2-hr recalculation window in dbt must correctly scope the delete range to avoid deleting hours outside the recalc window | Spot-check specific `(hour_bucket, zone_id)` pairs against Snowflake; verify no older hours disappear after a dbt run | +| `TRIPS_CDC_STREAM` / producer cutover | Migration script interrupted mid-run leaves a row-count gap; producer retry after cutover may re-insert trips already in ClickHouse | Run `scripts/01_verify_migration.sh` for row-count parity; `ReplacingMergeTree(_synced_at)` makes producer retries idempotent | + +
+ +--- + +## Transfer to migration-plan.md + +Copy your wave plan and complexity grades to Section 6 of `migration-plan.md` and check off: + +``` +- [ ] Migration wave plan: completed +``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md new file mode 100644 index 0000000..420263e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md @@ -0,0 +1,191 @@ +# Worksheet 5: dbt Model Design + +**Time estimate:** 15–20 minutes +**Reference:** [dbt on ClickHouse](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/dbt-on-clickhouse) + +--- + +## Concept + +Worksheets 1–4 produced the *what*: which engine, which ORDER BY, which ClickHouse types. This worksheet produces the *how*: how those decisions are expressed as dbt configuration before writing any SQL. + +A dbt-clickhouse model has three layers of configuration that Snowflake developers don't need to think about: + +1. **Materialization** — which physical object does dbt create (view, table, incremental, ephemeral)? +2. **Engine config** — for tables and incremental models, which ClickHouse engine and version column? +3. **Incremental strategy** — for incremental models, how does dbt handle new/updated rows? + +There is also a correctness concern unique to ReplacingMergeTree: + +4. **FINAL placement** — where in the dbt DAG does `FINAL` go to guarantee deduplicated reads? + +--- + +## Exercise 1: Materialization Selection + +For each model below, choose the correct dbt materialization type (`view`, `ephemeral`, `table`, or `incremental`) and write a one-sentence reasoning. + +Use the update pattern description to guide your choice: +- Insert-only read from a source? → `view` +- Pure join logic, not queried directly? → `ephemeral` +- Full rebuild on every dbt run, no partial updates? → `table` +- Only new/changed rows processed per run? → `incremental` + +Fill in the `?` cells: + +| Model | Layer | Update Pattern | Materialization | Why | +|-------|-------|----------------|-----------------|-----| +| `stg_trips` | staging | Read + clean from `trips_raw`; no writes to this model | ? | ? | +| `stg_taxi_zones` | staging | Read + clean from raw `taxi_zones` source (seeded by `scripts/00_seed_zones.sql`); no writes | ? | ? | +| `int_trips_enriched` | intermediate | Joins `stg_trips` with zone, payment, vendor, and date dims | ? | ? | +| `fact_trips` | analytics | One row per trip; trips can be corrected (fare adjustments re-insert same `trip_id`) | ? | ? | +| `agg_hourly_zone_trips` | analytics | dbt recalculates the last 2 hours on every run and re-inserts results | ? | ? | +| `dim_taxi_zones` | analytics | 265 static zones; dbt rebuilds the entire table from scratch on every run | ? | ? | +| `dim_payment_type` | analytics | 6 static types; dbt rebuilds from scratch on every run | ? | ? | +| `dim_vendor` | analytics | 3 vendors; dbt rebuilds from scratch on every run | ? | ? | + +--- + +## Exercise 2: Engine Configuration + +Only models with `table` or `incremental` materialization need a ClickHouse engine. Views and ephemeral models have no engine. + +For each model below, fill in the `ENGINE` and `Version Column` columns. Reference your answers from Worksheet 1 (engine selection) — the dbt engine config should match the design decisions you already made. + +| Model | Materialization | ENGINE | Version Column | Why this engine? | +|-------|----------------|--------|----------------|-----------------| +| `fact_trips` | incremental | ? | ? | ? | +| `agg_hourly_zone_trips` | incremental | ? | ? | ? | +| `dim_taxi_zones` | table | ? | — | ? | +| `dim_payment_type` | table | ? | — | ? | +| `dim_vendor` | table | ? | — | ? | + +**Hint:** Cross-check your answers against your completed Section 3 in `migration-plan.md`. The dbt engine config is the implementation of the engine decisions you made in Worksheet 1. + +--- + +## Exercise 3: Incremental Strategy Design + +For each incremental model, design the full incremental configuration: `unique_key`, `incremental_strategy`, and the SQL filter that goes inside the `is_incremental()` guard block. + +```sql +-- Template for the is_incremental() block: +{% if is_incremental() %} + WHERE +{% endif %} +``` + +Fill in the `?` cells: + +| Model | `unique_key` | `incremental_strategy` | Incremental filter (inside `is_incremental()`) | Why this filter? | +|-------|-------------|----------------------|------------------------------------------------|-----------------| +| `fact_trips` | ? | ? | `WHERE updated_at > ?` | ? | +| `agg_hourly_zone_trips` | ? | ? | `WHERE pickup_at >= ?` | ? | + +**Hints:** +- `fact_trips` uses a high-watermark pattern — it processes rows newer than the latest row already in the target. Which column should the watermark be on: `pickup_at` or `updated_at`? Think about what happens when a fare adjustment re-inserts the same `trip_id`. +- `agg_hourly_zone_trips` produces hourly aggregates. If you used a high-watermark like `fact_trips`, what would happen to the boundary hour that was partially calculated in the previous run? +- For `agg_hourly_zone_trips`, why is the composite key `[hour_bucket, zone_id]` correct here, while `fact_trips` uses only `trip_id`? + +--- + +## Exercise 4: FINAL Placement + +ReplacingMergeTree deduplication is asynchronous. `FINAL` forces it at read time — but placing FINAL in the wrong model has correctness or performance consequences. + +For each model below, decide whether `FINAL` should appear in the model's `FROM` clause (i.e., `FROM source FINAL` or `FROM {{ ref(...) }} FINAL`), and explain why: + +| Model | Reads from | FINAL in this model's FROM clause? | Why? | +|-------|-----------|-------------------------------------|------| +| `stg_trips` | `trips_raw` (ReplacingMergeTree) | ? | ? | +| `int_trips_enriched` | `stg_trips` (view) | ? | ? | +| `fact_trips` | `int_trips_enriched` (ephemeral) | ? | ? | + +**Hints:** +- `trips_raw` is ReplacingMergeTree — it can have duplicate `trip_id` rows from migration script retries or post-cutover producer retries. Where should this be resolved? +- `stg_trips` is a view, not an RMT table. Does reading from a view require FINAL? +- `fact_trips` uses `delete_insert`. Does that mean the table is already clean without FINAL? + +--- + +## Reflection Questions + +Answer these before checking the answer key: + +1. `int_trips_enriched` could be a `view` materialization (it reads from staging views). Why is `ephemeral` a better choice here? + +2. `agg_hourly_zone_trips` recalculates using `now() - INTERVAL 2 HOUR`. What would happen if you used `max(pickup_at)` instead (the same pattern as `fact_trips`)? + +3. You have `delete_insert` on `fact_trips` and FINAL in `stg_trips`. What would happen if you removed FINAL from `stg_trips` but kept delete_insert on `fact_trips`? + +--- + +
+▶ Answer Key — try the exercises first before expanding + +### Exercise 1: Materialization Selection + +| Model | Materialization | Why | +|-------|-----------------|-----| +| `stg_trips` | `view` | Reads and cleans source data; no updates to this model; views have zero storage cost and are always fresh | +| `stg_taxi_zones` | `view` | Same as stg_trips — passthrough cleaning of a source table | +| `int_trips_enriched` | `ephemeral` | Pure join logic consumed only by `fact_trips`; no need for a physical table; inlined as a CTE avoids an unnecessary database object | +| `fact_trips` | `incremental` | Trips can be corrected after the fact; only new and updated rows should be processed per run | +| `agg_hourly_zone_trips` | `incremental` | Rolling 2-hour recalculation is an incremental pattern — process recent rows, not all 50M | +| `dim_taxi_zones` | `table` | 265 static zones; full rebuild on every dbt run; no partial updates needed | +| `dim_payment_type` | `table` | 6 static types; same reasoning as dim_taxi_zones | +| `dim_vendor` | `table` | 3 vendors; same reasoning | + +### Exercise 2: Engine Configuration + +| Model | ENGINE | Version Column | Why | +|-------|--------|----------------|-----| +| `fact_trips` | `ReplacingMergeTree(updated_at)` | `updated_at` | Trips can be corrected; `updated_at` set to `now()` on each insert means the latest correction wins during RMT dedup | +| `agg_hourly_zone_trips` | `ReplacingMergeTree(updated_at)` | `updated_at` | Rolling recalculation re-inserts aggregates for (hour_bucket, zone_id) pairs; RMT ensures stale aggregates are removed on background merge | +| `dim_taxi_zones` | `MergeTree()` | — | Full reload by dbt means atomic table swap (full rebuild) on every run; no duplicate rows can accumulate; no dedup needed | +| `dim_payment_type` | `MergeTree()` | — | Same as dim_taxi_zones | +| `dim_vendor` | `MergeTree()` | — | Same as dim_taxi_zones | + +### Exercise 3: Incremental Strategy Design + +| Model | `unique_key` | `incremental_strategy` | Incremental filter | Why | +|-------|-------------|----------------------|-------------------|-----| +| `fact_trips` | `trip_id` | `delete_insert` | `WHERE updated_at > (SELECT max(updated_at) FROM {{ this }})` | High-watermark on `updated_at` processes both new trips and corrected trips (fare adjustments re-insert the same `trip_id` with the same `pickup_at` but a newer `updated_at`). A `pickup_at` watermark would silently miss corrections — the pickup time never changes. | +| `agg_hourly_zone_trips` | `[hour_bucket, zone_id]` | `delete_insert` | `WHERE pickup_at >= now() - INTERVAL 2 HOUR` | Rolling window re-aggregates the last 2 hours on every run; this ensures the current boundary hour is fully recalculated even if only partial data was available at the previous run's cutoff | + +**Why `[hour_bucket, zone_id]` for agg, not just `trip_id`?** +`agg_hourly_zone_trips` has one row per `(hour_bucket, zone_id)` pair, not one row per trip. The composite key `[hour_bucket, zone_id]` is the natural unique key. `delete_insert` deletes all (hour_bucket, zone_id) pairs that appear in the new batch, then inserts the freshly computed aggregates. + +**Why `now() - INTERVAL 2 HOUR` breaks with `max(pickup_at)`?** +If `agg_hourly_zone_trips` used `max(pickup_at)` as its filter, it would process only trips with `pickup_at > (last run's max)`. Trips from the current incomplete hour would be counted in one run, then never re-aggregated — the partial hour count would be permanently wrong. The rolling 2-hour window forces re-computation of recent hours so that boundary hours are always complete. + +### Exercise 4: FINAL Placement + +| Model | FINAL in FROM clause? | Why | +|-------|----------------------|-----| +| `stg_trips` | **Yes** — `FROM trips_raw FINAL` | `trips_raw` is ReplacingMergeTree; it can have duplicate `trip_id` rows from the bulk-load + CDC overlap window. `stg_trips` is the single enforcement point: deduplicate here so every downstream model sees clean data | +| `int_trips_enriched` | **No** | `int_trips_enriched` reads from `stg_trips`, which is a view (not an RMT table). FINAL is irrelevant for views. | +| `fact_trips` | **No** (in the model itself) | `delete_insert` keeps `fact_trips` clean — there should be no duplicates after a completed run. Dashboards and dbt tests that query `fact_trips` directly use `FINAL` externally as a safety net. Adding FINAL inside the model's SELECT would also apply it to the `is_incremental()` subquery that reads `max(pickup_at)` from `{{ this }}` — unnecessary overhead on every incremental run. | + +### Reflection Answers + +1. **Why ephemeral over view for `int_trips_enriched`?** +A view would create a database object that materializes the JOIN on every downstream query. Since `int_trips_enriched` is only used by `fact_trips`, an ephemeral model inlines the JOIN as a CTE directly inside `fact_trips`'s query plan — one execution, no extra round-trips, no physical object to maintain. + +2. **What breaks with `max(pickup_at)` for `agg_hourly_zone_trips`?** +With a high-watermark filter, only trips with `pickup_at > last_run_max` would be processed. Trips arriving in the current incomplete hour were partially counted in the previous run (up to `last_run_max`), then frozen. When new trips arrive for that same hour, they would not be re-aggregated — the boundary hour's count would be permanently understated. The rolling 2-hour window forces re-computation of recent hours so partial hours are always corrected. + +3. **What happens if you remove FINAL from `stg_trips`?** +Without FINAL, `stg_trips` may return two rows for the same `trip_id` if `trips_raw` has an unmerged duplicate from the bulk + CDC overlap. `int_trips_enriched` would join those two rows, producing two enriched rows for the same trip. `delete_insert` on `fact_trips` would then delete and re-insert both — so `fact_trips` would also have two rows for that `trip_id`. They would eventually be deduplicated by RMT's background merge, but until then, SUM(fare_amount) would double-count corrected trips and `dbt test` with `unique` on `trip_id` would fail. + +
+ +--- + +## Transfer to migration-plan.md + +Once you have completed this worksheet, fill in Section 10 of `migration-plan.md` with your answers and check off: + +``` +- [ ] dbt model design: completed +``` diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.env.example b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.env.example new file mode 100644 index 0000000..bf18b23 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.env.example @@ -0,0 +1,31 @@ +# ============================================================ +# .env.example — Part 3: ClickHouse Cloud credentials +# Add these to your existing .env from Part 1, or source both. +# NEVER commit .env to source control. +# +# Connection details for CLICKHOUSE_HOST and CLICKHOUSE_PORT +# are auto-generated by setup.sh into .clickhouse_state. +# Source both files in any terminal: +# source .env && source .clickhouse_state +# ============================================================ + +# ClickHouse Cloud — Terraform provisioning +export CLICKHOUSE_ORG_ID=... # Organization ID from https://console.clickhouse.cloud +export CLICKHOUSE_TOKEN_KEY=... # API token key (Settings → API Keys) +export CLICKHOUSE_TOKEN_SECRET=... # API token secret + +# ClickHouse Cloud — service credentials +export CLICKHOUSE_USER=default +export CLICKHOUSE_PASSWORD=... # Password for the ClickHouse service + +# Lab metadata (same as Part 1) +export LAB_COHORT=fy27-q1 # Used to name the ClickHouse service + +# Python migration script dependencies (install into .venv): +# pip install snowflake-connector-python clickhouse-connect +# Snowflake credentials are already in Part 1's .env: +# SNOWFLAKE_ORG, SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PASSWORD + +# Superset (from Part 1 — only needed if you are NOT sourcing Part 1's .env) +# export SUPERSET_ADMIN_USER=admin +# export SUPERSET_ADMIN_PASSWORD=admin diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.gitignore b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.gitignore new file mode 100644 index 0000000..1319f5e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/.gitignore @@ -0,0 +1,8 @@ +.env +.clickhouse_state +terraform/terraform.tfvars +terraform/.terraform/ +terraform/tfplan +terraform/terraform.tfstate +terraform/terraform.tfstate.backup +docs/part2-design-module-plan.md \ No newline at end of file diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/dbt_project.yml b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/dbt_project.yml new file mode 100644 index 0000000..43a8d6d --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/dbt_project.yml @@ -0,0 +1,60 @@ +name: 'nyc_taxi_dbt_ch' +version: '1.0.0' +config-version: 2 + +profile: 'nyc_taxi_ch' + +model-paths: ["models"] +test-paths: ["tests"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: ["target", "dbt_packages"] + +# ────────────────────────────────────────────────────────────────────────────── +# Migration note: Snowflake Part 1 used uppercase schema names (STAGING, ANALYTICS). +# ClickHouse schemas are lowercase — all schema references use | lower filter. +# ────────────────────────────────────────────────────────────────────────────── + +models: + nyc_taxi_dbt_ch: + staging: + +schema: staging + +materialized: view + # Note: no +engine here — views have no engine in ClickHouse. + # If you change a staging model to +materialized: table, add + # engine: "MergeTree()" and order_by directly in the model's config block. + + intermediate: + +schema: staging + +materialized: ephemeral + # ephemeral = CTEs inlined into downstream queries — same behavior as Snowflake Part 1 + + marts: + +schema: analytics + +materialized: table + +engine: "MergeTree()" + # Migration note: Snowflake used cluster_by for physical partitioning. + # ClickHouse uses order_by — the primary key index IS the physical sort order. + + fact_trips: + +materialized: incremental + # Migration note: Snowflake used incremental_strategy='merge' (MERGE INTO). + # ClickHouse has no MERGE INTO — ReplacingMergeTree handles deduplication + # via background merges. The version column (updated_at) determines which row wins. + # delete_insert is the correct dbt-clickhouse strategy for ReplacingMergeTree. + +engine: "ReplacingMergeTree(updated_at)" + +incremental_strategy: delete_insert + +unique_key: trip_id + + agg_hourly_zone_trips: + +materialized: incremental + +engine: "ReplacingMergeTree(updated_at)" + +incremental_strategy: delete_insert + +unique_key: [hour_bucket, zone_id] + + materialized_views: + +schema: analytics + +materialized: materialized_view + # Migration note: Snowflake has no native materialized views with auto-refresh. + # ClickHouse Materialized Views update on every INSERT — zero maintenance required. diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/macros/generate_schema_name.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/macros/generate_schema_name.sql new file mode 100644 index 0000000..342b21e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/macros/generate_schema_name.sql @@ -0,0 +1,19 @@ +-- Override dbt's default schema naming so models with a custom schema +-- land directly in that schema rather than being prefixed with target.schema. +-- +-- Example without this macro: target=nyc_taxi_ch + schema=analytics → "nyc_taxi_ch_analytics" +-- Example with this macro: target=nyc_taxi_ch + schema=analytics → "analytics" +-- +-- Why drop the prefix? In ClickHouse, schemas ARE databases. We want clean +-- database names (analytics, staging) not compound names (nyc_taxi_ch_analytics). +-- This matches how the migrated source data is organized. +-- +-- Snowflake equivalent: generate_schema_name macro (identical pattern, different reason — +-- Snowflake uses it to avoid STAGING_ANALYTICS schema name prefixing). +{% macro generate_schema_name(custom_schema_name, node) -%} + {%- if custom_schema_name is none -%} + {{ target.schema | lower }} + {%- else -%} + {{ custom_schema_name | lower }} + {%- endif -%} +{%- endmacro %} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/intermediate/int_trips_enriched.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/intermediate/int_trips_enriched.sql new file mode 100644 index 0000000..be4758b --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/intermediate/int_trips_enriched.sql @@ -0,0 +1,95 @@ +{{ + config( + materialized = 'ephemeral' + ) +}} +{# + Migration note: ephemeral materialization is identical in behavior between + Snowflake and ClickHouse dbt adapters — the model is inlined as a CTE into + any downstream model that references it. No translation needed here. +#} + +-- ════════════════════════════════════════════════════════════════════════════ +-- int_trips_enriched — Intermediate: trips joined to all dimension tables +-- +-- SNOWFLAKE → CLICKHOUSE TRANSLATION SUMMARY: +-- pickup_at::DATE = d.date_day → toDate(pickup_at) = d.date_day +-- Fully qualified Snowflake refs (NYC_TAXI_DB.ANALYTICS.DIM_*) +-- -> dbt ref() expressions pointing to local ClickHouse models +-- +-- All join logic and column selection is otherwise identical to Part 1. +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT + -- Core trip fields + t.trip_id, + t.pickup_at, + t.dropoff_at, + t.duration_minutes, + t.trip_distance_miles, + t.total_amount_usd, + t.tip_amount_usd, + t.fare_amount_usd, + t.extra_amount_usd, + t.mta_tax_usd, + t.tolls_amount_usd, + t.passenger_count, + + -- JSON-extracted metadata fields (already typed from stg_trips) + t.driver_rating, + t.driver_trips_completed, + t.vehicle_type, + t.app_platform, + t.app_version, + t.surge_multiplier, + t.traffic_level, + t.rate_code_id, + t.store_fwd_flag, + t.ingested_at, + + -- Zone enrichment — pickup + pu.borough AS pickup_borough, + pu.zone AS pickup_zone, + pu.service_zone AS pickup_service_zone, + + -- Zone enrichment — dropoff + do_.borough AS dropoff_borough, + do_.zone AS dropoff_zone, + do_.service_zone AS dropoff_service_zone, + + -- Payment type enrichment + pt.payment_code AS payment_code, + pt.payment_desc AS payment_type, + + -- Vendor enrichment + v.vendor_code AS vendor_code, + v.vendor_name AS vendor_name, + + -- Date dimension enrichment + d.day_of_week AS pickup_day_of_week, + d.day_of_week_num AS pickup_day_of_week_num, + d.month_name AS pickup_month, + d.quarter_num AS pickup_quarter, + d.fiscal_quarter AS fiscal_quarter, + d.is_weekend AS is_weekend, + d.is_holiday AS is_holiday, + d.holiday_name AS holiday_name + +FROM {{ ref('stg_trips') }} t + +LEFT JOIN {{ ref('stg_taxi_zones') }} pu + ON t.pickup_location_id = pu.location_id + +LEFT JOIN {{ ref('stg_taxi_zones') }} do_ + ON t.dropoff_location_id = do_.location_id + +LEFT JOIN {{ ref('dim_payment_type') }} pt + ON t.payment_type_id = pt.payment_type_id + +LEFT JOIN {{ ref('dim_vendor') }} v + ON t.vendor_id = v.vendor_id + +LEFT JOIN {{ ref('dim_date') }} d + -- Snowflake: ON date_spine.date_day::DATE = d.date_day + -- ClickHouse: toDate() extracts the date component from a DateTime column + ON toDate(t.pickup_at) = d.date_day diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/agg_hourly_zone_trips.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/agg_hourly_zone_trips.sql new file mode 100644 index 0000000..f06595b --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/agg_hourly_zone_trips.sql @@ -0,0 +1,70 @@ +{{ + config( + materialized = 'incremental', + unique_key = ['hour_bucket', 'zone_id'], + incremental_strategy = 'delete_insert', + engine = 'ReplacingMergeTree(updated_at)', + order_by = '(hour_bucket, zone_id)', + schema = 'analytics' + ) +}} +{# + Migration note: MERGE INTO → ReplacingMergeTree + delete_insert + See fact_trips.sql for full explanation of this pattern. + For aggregates, the delete_insert strategy: + 1. Deletes all rows whose (hour_bucket, zone_id) appear in the new batch + 2. Inserts the freshly recomputed aggregates for those keys + This correctly handles re-aggregation of partial hours at the window boundary. +#} + +-- ════════════════════════════════════════════════════════════════════════════ +-- agg_hourly_zone_trips — Pre-aggregated hourly trip metrics per zone +-- +-- SNOWFLAKE → CLICKHOUSE TRANSLATION SUMMARY: +-- DATE_TRUNC('hour', pickup_at) → toStartOfHour(pickup_at) +-- DATEADD('hour', -2, CURRENT_TIMESTAMP()) → now() - INTERVAL 2 HOUR +-- CURRENT_TIMESTAMP() → now() +-- COUNT(*) → count() +-- SUM() / AVG() → sum() / avg() (same, lowercase) +-- incremental_strategy='merge' → incremental_strategy='delete_insert' +-- MERGE INTO ... WHEN MATCHED THEN UPDATE → ReplacingMergeTree(updated_at) +-- +-- WHY delete_insert IS CORRECT HERE: +-- Snowflake's MERGE INTO found matching (hour_bucket, zone_id) rows and updated +-- the aggregate columns in place. ClickHouse tables are immutable on disk — +-- you cannot UPDATE in place. delete_insert achieves the same semantic: +-- delete stale aggregates for the affected keys, then insert fresh ones. +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT + -- Snowflake: DATE_TRUNC('hour', pickup_at) + -- ClickHouse: toStartOfHour() — purpose-built function, equivalent result + toStartOfHour(pickup_at) AS hour_bucket, + + pickup_location_id AS zone_id, + + -- Snowflake: COUNT(*) + -- ClickHouse: count() — the * is optional and conventional to omit + count() AS trips, + + sum(total_amount_usd) AS revenue, + avg(trip_distance_miles) AS avg_distance, + + -- Snowflake: CURRENT_TIMESTAMP() + -- ClickHouse: now() — returns current DateTime, same semantics + now() AS updated_at + +-- Source: stg_trips directly (bypasses int_trips_enriched for performance) +-- This model only needs pickup_at, pickup_location_id, total_amount_usd, trip_distance_miles. +-- Using the staging view avoids the 10-way join in int_trips_enriched for this aggregate. +-- Compare to fact_trips, which uses int_trips_enriched for the full denormalized row. +FROM {{ ref('stg_trips') }} + +{% if is_incremental() %} + -- Snowflake: WHERE pickup_at >= DATEADD('hour', -2, CURRENT_TIMESTAMP()) + -- ClickHouse: interval arithmetic uses INTERVAL keyword with unit noun + -- 'day'/DATEADD → today() - INTERVAL N DAY | 'hour' → now() - INTERVAL N HOUR + WHERE pickup_at >= now() - INTERVAL 2 HOUR +{% endif %} + +GROUP BY 1, 2 diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_date.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_date.sql new file mode 100644 index 0000000..111dfc6 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_date.sql @@ -0,0 +1,153 @@ +{{ + config( + materialized = 'table', + engine = 'MergeTree()', + order_by = '(date_day)', + schema = 'analytics' + ) +}} + +-- ════════════════════════════════════════════════════════════════════════════ +-- dim_date — Date dimension: 2009-01-01 through 2029-12-31 +-- +-- SNOWFLAKE → CLICKHOUSE TRANSLATION SUMMARY: +-- dbt_utils.date_spine(...) -> numbers() table function +-- TO_CHAR(date_day, 'DY') → formatDateTime(date_day, '%a') +-- DAYOFWEEK(date_day) → toDayOfWeek(date_day) +-- MONTH(date_day) → toMonth(date_day) +-- TO_CHAR(date_day, 'MON') → formatDateTime(date_day, '%b') +-- QUARTER(date_day) → toQuarter(date_day) +-- YEAR(date_day) → toYear(date_day) +-- RIGHT(YEAR(...)::VARCHAR, 2) → substring(toString(toYear(...)), 3, 2) +-- CONCAT(...) → concat(...) (same name, lowercase) +-- TRUE / FALSE → 1 / 0 (ClickHouse uses UInt8 for booleans) +-- ::DATE cast on date_spine row → toDate() cast +-- VALUES (...) AS t (col, col) → VALUES ... without alias needed (inline CTE) +-- +-- KEY DIFFERENCE — date spine generation: +-- Snowflake: dbt_utils.date_spine() macro (generates a SELECT with UNION ALL or +-- recursive CTE depending on adapter) +-- ClickHouse: numbers(N) generates integers 0..N-1. Adding toIntervalDay(number) +-- to a base Date is the idiomatic ClickHouse approach. Fast: ~7000 rows, +-- all computed in a single pass with no recursion. +-- ════════════════════════════════════════════════════════════════════════════ + +-- US Federal holidays (static reference — same data as Part 1, translated syntax) +-- Migration note: Snowflake used VALUES (...::DATE, '...') with PostgreSQL-style cast. +-- ClickHouse uses toDate('YYYY-MM-DD') for explicit date literals. +WITH us_holidays AS ( + SELECT toDate('2019-01-01') AS holiday_date, 'New Year''s Day' AS holiday_name UNION ALL + SELECT toDate('2019-01-21'), 'MLK Day' UNION ALL + SELECT toDate('2019-02-18'), 'Presidents'' Day' UNION ALL + SELECT toDate('2019-05-27'), 'Memorial Day' UNION ALL + SELECT toDate('2019-07-04'), 'Independence Day' UNION ALL + SELECT toDate('2019-09-02'), 'Labor Day' UNION ALL + SELECT toDate('2019-11-11'), 'Veterans Day' UNION ALL + SELECT toDate('2019-11-28'), 'Thanksgiving' UNION ALL + SELECT toDate('2019-12-25'), 'Christmas' UNION ALL + -- 2020 + SELECT toDate('2020-01-01'), 'New Year''s Day' UNION ALL + SELECT toDate('2020-01-20'), 'MLK Day' UNION ALL + SELECT toDate('2020-02-17'), 'Presidents'' Day' UNION ALL + SELECT toDate('2020-05-25'), 'Memorial Day' UNION ALL + SELECT toDate('2020-07-04'), 'Independence Day' UNION ALL + SELECT toDate('2020-09-07'), 'Labor Day' UNION ALL + SELECT toDate('2020-11-11'), 'Veterans Day' UNION ALL + SELECT toDate('2020-11-26'), 'Thanksgiving' UNION ALL + SELECT toDate('2020-12-25'), 'Christmas' UNION ALL + -- 2021 + SELECT toDate('2021-01-01'), 'New Year''s Day' UNION ALL + SELECT toDate('2021-01-18'), 'MLK Day' UNION ALL + SELECT toDate('2021-02-15'), 'Presidents'' Day' UNION ALL + SELECT toDate('2021-05-31'), 'Memorial Day' UNION ALL + SELECT toDate('2021-07-05'), 'Independence Day (observed)' UNION ALL + SELECT toDate('2021-09-06'), 'Labor Day' UNION ALL + SELECT toDate('2021-11-11'), 'Veterans Day' UNION ALL + SELECT toDate('2021-11-25'), 'Thanksgiving' UNION ALL + SELECT toDate('2021-12-25'), 'Christmas' UNION ALL + -- 2022 + SELECT toDate('2022-01-01'), 'New Year''s Day' UNION ALL + SELECT toDate('2022-01-17'), 'MLK Day' UNION ALL + SELECT toDate('2022-02-21'), 'Presidents'' Day' UNION ALL + SELECT toDate('2022-05-30'), 'Memorial Day' UNION ALL + SELECT toDate('2022-07-04'), 'Independence Day' UNION ALL + SELECT toDate('2022-09-05'), 'Labor Day' UNION ALL + SELECT toDate('2022-11-11'), 'Veterans Day' UNION ALL + SELECT toDate('2022-11-24'), 'Thanksgiving' UNION ALL + SELECT toDate('2022-12-26'), 'Christmas (observed)' UNION ALL + -- 2023 + SELECT toDate('2023-01-02'), 'New Year''s Day (observed)' UNION ALL + SELECT toDate('2023-01-16'), 'MLK Day' UNION ALL + SELECT toDate('2023-02-20'), 'Presidents'' Day' UNION ALL + SELECT toDate('2023-05-29'), 'Memorial Day' UNION ALL + SELECT toDate('2023-07-04'), 'Independence Day' UNION ALL + SELECT toDate('2023-09-04'), 'Labor Day' UNION ALL + SELECT toDate('2023-11-10'), 'Veterans Day (observed)' UNION ALL + SELECT toDate('2023-11-23'), 'Thanksgiving' UNION ALL + SELECT toDate('2023-12-25'), 'Christmas' +), + +-- Migration note: Snowflake used dbt_utils.date_spine(datepart="day", ...) +-- which expands to a recursive CTE or UNION ALL depending on the adapter. +-- ClickHouse numbers(N) generates a column of UInt64 values [0, N-1] in one pass. +-- toDate('2009-01-01') + toIntervalDay(number) builds each date arithmetically. +-- dateDiff('day', start, end) computes the span — ~7670 rows for 2009–2029. +date_spine AS ( + SELECT + toDate('2009-01-01') + toIntervalDay(number) AS date_day + FROM numbers( + toUInt32(dateDiff('day', toDate('2009-01-01'), toDate('2029-12-31'))) + 1 + -- +1 because dateDiff() excludes the end date; we want the spine to include 2029-12-31 + ) +), + +enriched AS ( + SELECT + date_day, + + -- Snowflake: TO_CHAR(date_day, 'DY') e.g. 'Mon', 'Tue' + -- ClickHouse: formatDateTime with strftime-style format string + formatDateTime(date_day, '%a') AS day_of_week, + + -- Snowflake: DAYOFWEEK(date_day) (0=Sunday in Snowflake) + -- ClickHouse: toDayOfWeek(date_day) (1=Monday by default, ISO 8601) + -- Note: ClickHouse toDayOfWeek returns 1(Mon)–7(Sun). Snowflake returns 0(Sun)–6(Sat). + -- Use toDayOfWeek(date_day, 0) for Sunday=0 mode to match Snowflake exactly. + toDayOfWeek(date_day, 0) AS day_of_week_num, + + -- Snowflake: MONTH(date_day) + toMonth(date_day) AS month_num, + + -- Snowflake: TO_CHAR(date_day, 'MON') e.g. 'Jan', 'Feb' + formatDateTime(date_day, '%b') AS month_name, + + -- Snowflake: QUARTER(date_day) + toQuarter(date_day) AS quarter_num, + + -- Snowflake: YEAR(date_day) + toYear(date_day) AS year_num, + + -- Snowflake: CONCAT('FY', RIGHT(YEAR(date_day)::VARCHAR, 2), 'Q', QUARTER(date_day)) + -- ClickHouse: concat() + toString() + substring() for right-2-chars of year + concat( + 'FY', + substring(toString(toYear(date_day)), 3, 2), + 'Q', + toString(toQuarter(date_day)) + ) AS fiscal_quarter, + + -- Snowflake: CASE WHEN DAYOFWEEK(date_day) IN (0, 6) THEN TRUE ELSE FALSE END + -- ClickHouse: 1/0 UInt8 (no native BOOLEAN type; ClickHouse uses UInt8) + -- toDayOfWeek(date_day, 0): 0=Sunday, 6=Saturday — matching Snowflake semantics + if(toDayOfWeek(date_day, 0) IN (0, 6), 1, 0) AS is_weekend, + + -- Snowflake: CASE WHEN h.holiday_date IS NOT NULL THEN TRUE ELSE FALSE END + if(h.holiday_date IS NOT NULL, 1, 0) AS is_holiday, + + h.holiday_name + + FROM date_spine + LEFT JOIN us_holidays h ON date_spine.date_day = h.holiday_date +) + +SELECT * FROM enriched diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_payment_type.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_payment_type.sql new file mode 100644 index 0000000..0a2ca2e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_payment_type.sql @@ -0,0 +1,36 @@ +{{ + config( + materialized = 'table', + engine = 'MergeTree()', + order_by = '(payment_type_id)', + schema = 'analytics' + ) +}} + +-- ════════════════════════════════════════════════════════════════════════════ +-- dim_payment_type — Payment type lookup (static seed data) +-- +-- Migration note: In Snowflake Part 1, payment types were inserted via the +-- 01_create_tables.sql seed script: +-- INSERT INTO DIM_PAYMENT_TYPE VALUES (1, 'CREDIT', 'Credit Card'), ... +-- and dim_payment_type.sql referenced NYC_TAXI_DB.ANALYTICS.DIM_PAYMENT_TYPE. +-- +-- In ClickHouse, the Python migration script only migrates trip data; zone data is +-- seeded separately. Payment type is static reference data that doesn't exist in +-- any raw table, so we embed the VALUES directly in the model. This creates a +-- self-contained dbt project with no external seed dependencies for this dimension. +-- +-- SQL translation: +-- INSERT INTO ... VALUES → SELECT * FROM (VALUES ...) as inline CTE +-- No Snowflake-specific syntax — VALUES is ANSI SQL, supported in ClickHouse. +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT * +FROM ( + SELECT 1 AS payment_type_id, 'CREDIT' AS payment_code, 'Credit Card' AS payment_desc + UNION ALL SELECT 2, 'CASH', 'Cash' + UNION ALL SELECT 3, 'NO_CHG', 'No Charge' + UNION ALL SELECT 4, 'DISPUTE', 'Dispute' + UNION ALL SELECT 5, 'UNKNOWN', 'Unknown' + UNION ALL SELECT 6, 'VOIDED', 'Voided Trip' +) diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_taxi_zones.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_taxi_zones.sql new file mode 100644 index 0000000..a952905 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_taxi_zones.sql @@ -0,0 +1,29 @@ +{{ + config( + materialized = 'table', + engine = 'MergeTree()', + order_by = '(location_id)', + schema = 'analytics' + ) +}} + +-- ════════════════════════════════════════════════════════════════════════════ +-- dim_taxi_zones — Taxi zone dimension (analytics layer) +-- +-- Migration note: In Snowflake Part 1, zone data was seeded directly into +-- NYC_TAXI_DB.ANALYTICS.DIM_TAXI_ZONES via SQL seed scripts (01_create_tables.sql). +-- dim_taxi_zones was essentially a pass-through with no transformation. +-- +-- In ClickHouse, the source data flows through the staging layer: +-- scripts/00_seed_zones.sql → default.taxi_zones (raw source) → stg_taxi_zones → analytics.dim_taxi_zones +-- +-- This model simply promotes the staged/cleaned zones to the analytics schema. +-- No SQL translation required beyond the ref() path change. +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT + location_id, + borough, + zone, + service_zone +FROM {{ ref('stg_taxi_zones') }} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_vendor.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_vendor.sql new file mode 100644 index 0000000..648ce18 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/dim_vendor.sql @@ -0,0 +1,31 @@ +{{ + config( + materialized = 'table', + engine = 'MergeTree()', + order_by = '(vendor_id)', + schema = 'analytics' + ) +}} + +-- ════════════════════════════════════════════════════════════════════════════ +-- dim_vendor — Vendor lookup (static seed data) +-- +-- Migration note: Same pattern as dim_payment_type. In Snowflake Part 1, vendor +-- data was seeded via 01_create_tables.sql INSERT statements and referenced as +-- NYC_TAXI_DB.ANALYTICS.DIM_VENDOR. +-- +-- In ClickHouse, vendor data is embedded directly in the model as static VALUES. +-- This avoids any external seed file dependency for small static dimensions. +-- +-- These are the three TLC-registered taxi vendors in the NYC Taxi dataset: +-- CMT = Creative Mobile Technologies (app/dispatch system) +-- VTS = VeriFone Inc. (payment terminals) +-- DDS = Digital Dispatch Systems +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT * +FROM ( + SELECT 1 AS vendor_id, 'CMT' AS vendor_code, 'Creative Mobile Technologies' AS vendor_name + UNION ALL SELECT 2, 'VTS', 'VeriFone Inc.' + UNION ALL SELECT 3, 'DDS', 'Digital Dispatch Systems' +) diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/fact_trips.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/fact_trips.sql new file mode 100644 index 0000000..4591902 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/fact_trips.sql @@ -0,0 +1,111 @@ +{{ + config( + materialized = 'incremental', + unique_key = 'trip_id', + incremental_strategy = 'delete_insert', + engine = 'ReplacingMergeTree(updated_at)', + order_by = '(toStartOfMonth(pickup_at), pickup_at, trip_id)', + schema = 'analytics' + ) +}} +{# + Migration note: Snowflake used incremental_strategy='merge' which maps to MERGE INTO. + ClickHouse has no MERGE INTO statement. Instead: + 1. The table engine ReplacingMergeTree(updated_at) tracks a version column. + 2. On background merges, ClickHouse keeps only the row with the highest updated_at + per (order_by key). This replaces Snowflake MERGE INTO WHEN MATCHED THEN UPDATE. + 3. delete_insert is the dbt-clickhouse incremental strategy that: + a. DELETEs rows where unique_key matches the incoming batch + b. INSERTs the full new batch + This is the correct strategy for ReplacingMergeTree in dbt. + 4. At query time, use SELECT ... FINAL to force immediate deduplication + (e.g., SELECT * FROM fact_trips FINAL). Without FINAL, duplicate rows + from in-flight merges may appear briefly. + + Migration note: Snowflake used cluster_by=['pickup_at::DATE'] for physical clustering. + ClickHouse uses order_by as the primary key AND physical sort order — no separate + cluster_by concept. The order_by tuple defines the sparse primary index. +#} + +-- ┌─────────────────────────────────────────────────────────────────┐ +-- │ Migration Note: Snowflake MERGE INTO → ClickHouse Strategy │ +-- │ │ +-- │ Snowflake: MERGE INTO ... WHEN MATCHED THEN UPDATE │ +-- │ ClickHouse: dbt delete_insert incremental strategy │ +-- │ │ +-- │ How delete_insert works: │ +-- │ 1. dbt deletes rows matching unique_key (trip_id) from the │ +-- │ target table for the new batch │ +-- │ 2. dbt inserts the new batch │ +-- │ This is the primary mechanism ensuring correctness. │ +-- │ │ +-- │ Role of ReplacingMergeTree(updated_at): │ +-- │ - It is a SAFETY NET, not the primary deduplication path │ +-- │ - If a delete_insert run is interrupted mid-flight, the │ +-- │ ReplacingMergeTree engine will deduplicate duplicates │ +-- │ during the next background merge, keeping the row with │ +-- │ the highest updated_at value │ +-- │ - ClickHouse merges are EVENTUAL (async background process) │ +-- │ │ +-- │ For analytical queries requiring point-in-time correctness: │ +-- │ SELECT ... FROM analytics.fact_trips FINAL │ +-- │ FINAL forces synchronous deduplication at query time. │ +-- │ It adds latency but guarantees no duplicate trip_ids. │ +-- └─────────────────────────────────────────────────────────────────┘ + +-- ════════════════════════════════════════════════════════════════════════════ +-- fact_trips — Central fact table, 50M rows, one row per trip +-- +-- SNOWFLAKE → CLICKHOUSE TRANSLATION SUMMARY: +-- MERGE INTO (incremental) → ReplacingMergeTree + delete_insert +-- incremental_strategy='merge' → incremental_strategy='delete_insert' +-- cluster_by=['pickup_at::DATE'] → order_by='(toStartOfMonth(pickup_at), pickup_at, trip_id)' +-- CURRENT_TIMESTAMP() → now() +-- MAX(updated_at) incremental filter → max(updated_at) (lowercase) +-- now() AS updated_at → version column for ReplacingMergeTree +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT + trip_id, + pickup_at, + dropoff_at, + duration_minutes, + trip_distance_miles, + total_amount_usd, + tip_amount_usd, + fare_amount_usd, + extra_amount_usd, + mta_tax_usd, + tolls_amount_usd, + passenger_count, + driver_rating, + vehicle_type, + app_platform, + surge_multiplier, + traffic_level, + pickup_borough, + pickup_zone, + pickup_service_zone, + dropoff_borough, + dropoff_zone, + payment_type, + vendor_name, + pickup_day_of_week, + fiscal_quarter, + is_weekend, + is_holiday, + ingested_at, + -- Snowflake: now() AS updated_at (same function name, different case convention) + -- ClickHouse: now() returns DateTime — used as the version column by ReplacingMergeTree. + -- Rows with a higher updated_at value win during background deduplication merges. + now() AS updated_at + +FROM {{ ref('int_trips_enriched') }} + +{% if is_incremental() %} + -- High-watermark on updated_at (not pickup_at) so that fare adjustments are captured. + -- A corrected trip re-inserts the same trip_id with the same pickup_at but a newer + -- updated_at (set to now() on every insert). A pickup_at watermark would silently + -- miss those corrections — the pickup time never changes. + WHERE updated_at > (SELECT max(updated_at) FROM {{ this }}) +{% endif %} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/schema.yml b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/schema.yml new file mode 100644 index 0000000..9ae5267 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/marts/schema.yml @@ -0,0 +1,145 @@ +version: 2 + +# ────────────────────────────────────────────────────────────────────────────── +# Migration note: All dbt_expectations tests from Part 1 have been removed. +# dbt_expectations is not compatible with the dbt-clickhouse adapter. +# Range/value tests are covered by: +# - Custom SQL tests: tests/assert_revenue_positive.sql, tests/assert_trips_not_future.sql +# - Native dbt tests: not_null, unique, accepted_values +# ────────────────────────────────────────────────────────────────────────────── + +models: + - name: fact_trips + description: > + Central fact table — one row per trip, fully denormalized star schema. + ~50M rows. Incremental model using ReplacingMergeTree(updated_at) engine. + Migration note: Snowflake used MERGE INTO (incremental_strategy='merge'). + ClickHouse uses ReplacingMergeTree + delete_insert strategy. Query with + SELECT ... FINAL to force deduplication at query time. + columns: + - name: trip_id + description: "UUID string — unique key for deduplication (ReplacingMergeTree)" + tests: + - not_null + - unique + + - name: pickup_at + description: "Pickup DateTime — part of the order_by: toStartOfMonth(pickup_at) as coarse prefix, pickup_at for range scans, trip_id for RMT uniqueness" + tests: + - not_null + + - name: total_amount_usd + description: "Total fare amount in USD — must be non-negative (see assert_revenue_positive)" + tests: + - not_null + + - name: payment_type + description: "Payment type description (joined from dim_payment_type)" + tests: + - not_null + + - name: pickup_borough + description: "Pickup borough name — joined from dim_taxi_zones via stg_taxi_zones" + + - name: updated_at + description: > + Version column for ReplacingMergeTree. Set to now() on every insert. + Migration note: Not present in Snowflake fact_trips — added specifically + for ClickHouse deduplication semantics. + + - name: agg_hourly_zone_trips + description: > + Pre-aggregated hourly zone trip metrics. Updated incrementally. + Migration note: Snowflake used MERGE INTO to update aggregate rows in place. + ClickHouse uses delete_insert + ReplacingMergeTree: delete stale hour/zone + aggregates then insert freshly computed ones. + columns: + - name: hour_bucket + description: > + Truncated to hour boundary. + Migration note: DATE_TRUNC('hour', pickup_at) → toStartOfHour(pickup_at) + tests: + - not_null + + - name: zone_id + description: "TLC pickup location ID (part of composite unique key)" + tests: + - not_null + + - name: trips + description: "Count of trips in this hour/zone bucket" + tests: + - not_null + + - name: revenue + description: "Sum of total_amount_usd for this hour/zone bucket" + + - name: avg_distance + description: "Average trip_distance_miles for this hour/zone bucket" + + - name: updated_at + description: "Version column for ReplacingMergeTree — set to now() on each run" + + - name: dim_date + description: > + Date dimension spanning 2009-01-01 to 2029-12-31. Includes fiscal quarters, + day-of-week, US federal holiday flags. + Migration note: Snowflake used dbt_utils.date_spine() macro for date generation. + ClickHouse uses numbers() table function: far simpler, no package dependency, + and generates all ~7670 rows in a single scan. + columns: + - name: date_day + description: "Calendar date (Date type in ClickHouse)" + tests: + - not_null + - unique + + - name: fiscal_quarter + description: "Format FYyyQq e.g. FY23Q1 — same logic as Part 1, different concat syntax" + + - name: is_weekend + description: "1 if Saturday or Sunday, 0 otherwise (UInt8 — ClickHouse has no BOOLEAN)" + + - name: is_holiday + description: "1 if US federal holiday, 0 otherwise" + + - name: dim_taxi_zones + description: > + NYC TLC taxi zone lookup — 265 zones across 5 boroughs. + Migration note: In Snowflake, data was seeded via 01_create_tables.sql. + In ClickHouse, seeded via scripts/00_seed_zones.sql and promoted through stg_taxi_zones. + columns: + - name: location_id + description: "TLC location ID (1–265)" + tests: + - not_null + - unique + + - name: dim_payment_type + description: > + Payment type lookup — 6 types (static seed data embedded in model). + Migration note: In Snowflake, seeded via SQL INSERT. In ClickHouse, embedded + as VALUES in the model SELECT since it is not part of any migrated raw table. + columns: + - name: payment_type_id + tests: + - not_null + - unique + - name: payment_code + tests: + - not_null + - accepted_values: + values: ['CREDIT', 'CASH', 'NO_CHG', 'DISPUTE', 'UNKNOWN', 'VOIDED'] + + - name: dim_vendor + description: > + Vendor lookup — 3 NYC TLC registered taxi technology vendors (static). + Migration note: Same as dim_payment_type — embedded VALUES, no external seed. + columns: + - name: vendor_id + tests: + - not_null + - unique + - name: vendor_code + tests: + - not_null diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/materialized_views/mv_live_trip_feed.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/materialized_views/mv_live_trip_feed.sql new file mode 100644 index 0000000..57e4f01 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/materialized_views/mv_live_trip_feed.sql @@ -0,0 +1,38 @@ +{{ + config( + materialized = 'materialized_view', + schema = 'analytics', + engine = 'ReplacingMergeTree(refreshed_at)', + order_by = '(snapshot_key)', + ) +}} +-- ┌─────────────────────────────────────────────────────────────────┐ +-- │ ClickHouse-Exclusive Feature: REFRESHABLE Materialized View │ +-- │ │ +-- │ Snowflake equivalent: None. Closest is a Snowflake Task that │ +-- │ runs a stored procedure on a schedule — but that's 30+ lines │ +-- │ of TASK DDL. In ClickHouse, one ALTER TABLE statement does it. │ +-- │ │ +-- │ Why REFRESHABLE, not a standard trigger-based MV? │ +-- │ Standard ClickHouse MVs fire on INSERT and see only the batch │ +-- │ being inserted — not the full table. They cannot compute │ +-- │ lifetime aggregates like total_trips or avg fare correctly. │ +-- │ REFRESHABLE MVs do a full re-scan on a schedule (23.4+). │ +-- │ │ +-- │ After dbt run, enable periodic refresh with: │ +-- │ ALTER TABLE analytics.mv_live_trip_feed │ +-- │ MODIFY REFRESH EVERY 30 SECOND; │ +-- │ │ +-- │ Query the latest snapshot: │ +-- │ SELECT * FROM analytics.mv_live_trip_feed FINAL │ +-- │ ORDER BY refreshed_at DESC LIMIT 1; │ +-- └─────────────────────────────────────────────────────────────────┘ +SELECT + 1 AS snapshot_key, -- fixed key so FINAL deduplicates to 1 row + now() AS refreshed_at, + count() AS total_trips, + countIf(pickup_at >= now() - INTERVAL 1 HOUR) AS trips_last_hour, + countIf(toDate(pickup_at) = today()) AS trips_today, + round(avg(total_amount_usd), 2) AS avg_fare_usd, + round(sumIf(total_amount_usd, toDate(pickup_at) = today()), 2) AS revenue_today +FROM {{ ref('fact_trips') }} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/sources.yml b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/sources.yml new file mode 100644 index 0000000..419f046 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/sources.yml @@ -0,0 +1,66 @@ +version: 2 + +# ────────────────────────────────────────────────────────────────────────────── +# Migration note: In Snowflake Part 1, sources pointed to NYC_TAXI_DB.RAW schema. +# In ClickHouse, raw data is loaded via the Python migration script +# (scripts/02_migrate_trips.py) and live writes from the post-cutover producer. +# Both tables land in the default database/schema. +# ────────────────────────────────────────────────────────────────────────────── + +sources: + - name: raw + database: default + schema: default + description: "Raw NYC Taxi data — loaded from Snowflake via Python migration script" + tables: + - name: trips_raw + description: > + All 50M trip records with a JSON string metadata column (trip_metadata). + Migration note: In Snowflake this was TRIPS_RAW with a VARIANT column. + In ClickHouse the column is stored as String and parsed with JSONExtract functions. + columns: + - name: trip_id + description: "UUID string — primary key for deduplication" + tests: + - not_null + - name: pickup_at + description: "Pickup timestamp — DateTime64(3,'UTC') in ClickHouse (was TIMESTAMP_NTZ in Snowflake)" + tests: + - not_null + - name: dropoff_at + description: "Dropoff timestamp" + - name: trip_metadata + description: > + JSON string containing nested driver, app, and route metadata. + Migration note: In Snowflake this was a VARIANT column accessed via + colon-path syntax (TRIP_METADATA:driver.rating::FLOAT). + In ClickHouse, use JSONExtractFloat(), JSONExtractString(), JSONExtractInt(). + - name: _synced_at + description: > + ClickHouse-side ingestion timestamp — set automatically to now() on every INSERT + via DEFAULT now(). Used as the ReplacingMergeTree(_synced_at) version column. + If the migration script is interrupted and re-run with --resume, duplicate rows + for the same trip_id may briefly exist; the later INSERT has a higher _synced_at + and wins during deduplication. Post-cutover producer retries are also safe for + the same reason. stg_trips queries trips_raw FINAL to force dedup before + downstream models run. + + - name: taxi_zones + description: > + NYC TLC taxi zone lookup — 265 zones across 5 boroughs. + Migration note: In Snowflake this was seeded into ANALYTICS.DIM_TAXI_ZONES. + In ClickHouse it is seeded once via scripts/00_seed_zones.sql as default.taxi_zones + before the first dbt run. Named taxi_zones (not dim_taxi_zones) to avoid naming + collision with the analytics.dim_taxi_zones dbt model. It is static reference data + and does not change. + columns: + - name: location_id + description: "TLC location ID (1–265)" + tests: + - not_null + - name: borough + description: "NYC borough name" + - name: zone + description: "Named zone within borough" + - name: service_zone + description: "TLC service zone (Yellow Zone, Boro Zone, EWR)" diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/schema.yml b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/schema.yml new file mode 100644 index 0000000..1f695a1 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/schema.yml @@ -0,0 +1,99 @@ +version: 2 + +# ────────────────────────────────────────────────────────────────────────────── +# Migration note: Part 1 (Snowflake) used dbt_expectations for range tests: +# dbt_expectations.expect_column_values_to_be_between (min/max) +# dbt_expectations is NOT compatible with the dbt-clickhouse adapter. +# Equivalent coverage is provided by: +# - Native dbt tests: not_null, unique, accepted_values +# - Custom SQL tests: tests/assert_revenue_positive.sql +# ────────────────────────────────────────────────────────────────────────────── + +models: + - name: stg_trips + description: > + Cleaned and typed staging layer for NYC Taxi trip records. + VARIANT/JSON metadata column flattened to typed columns using JSONExtract functions. + Migration note: replaces Snowflake colon-path VARIANT syntax with + JSONExtractFloat(), JSONExtractInt(), JSONExtractString(). + columns: + - name: trip_id + description: "UUID string — primary key for deduplication" + tests: + - not_null + - unique + + - name: pickup_at + description: "Pickup timestamp — DateTime in ClickHouse (TIMESTAMP_NTZ in Snowflake)" + tests: + - not_null + + - name: dropoff_at + description: "Dropoff timestamp" + tests: + - not_null + + - name: total_amount_usd + description: "Total fare including all components" + tests: + - not_null + # Migration note: Part 1 used dbt_expectations.expect_column_values_to_be_between + # (min_value: 0, max_value: 1000). Replaced by custom test assert_revenue_positive.sql + + - name: duration_minutes + description: > + Trip duration in minutes. + Migration note: Derived via dateDiff('minute', pickup_at, dropoff_at) + — Snowflake equivalent was DATEDIFF('minute', PICKUP_DATETIME, DROPOFF_DATETIME). + # Migration note: Part 1 used dbt_expectations range test (min 1, max 600). + # Covered by WHERE dropoff_at > pickup_at filter in the model instead. + + - name: payment_type_id + description: "Payment type code (1=Credit, 2=Cash, 3=No Charge, 4=Dispute, 5=Unknown, 6=Voided)" + tests: + - accepted_values: + values: [1, 2, 3, 4, 5, 6] + + - name: driver_rating + description: > + Driver rating extracted from trip_metadata JSON. + Migration note: In Snowflake — TRIP_METADATA:driver.rating::FLOAT + In ClickHouse — JSONExtractFloat(trip_metadata, 'driver', 'rating') + + - name: surge_multiplier + description: > + App surge pricing factor from JSON metadata. 1.0 = no surge. + Migration note: In Snowflake — TRIP_METADATA:app.surge_multiplier::FLOAT + In ClickHouse — JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier') + + - name: vehicle_type + description: "Vehicle type string from JSON metadata (e.g. 'SUV', 'Sedan')" + + - name: app_platform + description: "App platform from JSON metadata (e.g. 'iOS', 'Android')" + + - name: traffic_level + description: "Traffic level from JSON metadata (e.g. 'light', 'moderate', 'heavy')" + + - name: stg_taxi_zones + description: > + Cleaned taxi zone dimension from TLC lookup table. + Migration note: In Snowflake, source was NYC_TAXI_DB.ANALYTICS.DIM_TAXI_ZONES. + In ClickHouse, seeded via scripts/00_seed_zones.sql as source('raw', 'taxi_zones'). + columns: + - name: location_id + description: "TLC location ID (1–265)" + tests: + - not_null + - unique + + - name: borough + description: "NYC borough — defaults to 'Unknown' via coalesce()" + tests: + - not_null + + - name: zone + description: "Named zone within borough — defaults to 'Unknown'" + + - name: service_zone + description: "TLC service zone (Yellow Zone, Boro Zone, EWR) — defaults to 'Unknown'" diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_taxi_zones.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_taxi_zones.sql new file mode 100644 index 0000000..55a648a --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_taxi_zones.sql @@ -0,0 +1,36 @@ +-- ════════════════════════════════════════════════════════════════════════════ +-- stg_taxi_zones — Cleaned taxi zone dimension +-- +-- Migration note: In Snowflake Part 1, zone data was pre-seeded into +-- NYC_TAXI_DB.ANALYTICS.DIM_TAXI_ZONES via SQL seed scripts, and stg_taxi_zones +-- queried it directly with a fully qualified name: +-- FROM NYC_TAXI_DB.ANALYTICS.DIM_TAXI_ZONES +-- +-- In ClickHouse, the zone lookup table is seeded once via scripts/00_seed_zones.sql +-- into default.taxi_zones. We reference it via source('raw', 'taxi_zones'). +-- (Named taxi_zones, not dim_taxi_zones, to avoid confusion with analytics.dim_taxi_zones.) +-- +-- SQL translation: +-- COALESCE() → coalesce() (same function, lowercase in ClickHouse convention) +-- location_id IS NOT NULL → location_id != 0 (Int type, not nullable in MergeTree) +-- ════════════════════════════════════════════════════════════════════════════ + +{{ + config( + materialized = 'view', + schema = 'staging' + ) +}} + +SELECT + location_id, + -- Snowflake: COALESCE(BOROUGH, 'Unknown') + -- ClickHouse: coalesce() works identically; column names are lowercase + coalesce(borough, 'Unknown') AS borough, + coalesce(zone, 'Unknown') AS zone, + coalesce(service_zone, 'Unknown') AS service_zone +FROM {{ source('raw', 'taxi_zones') }} + +-- Snowflake: WHERE LOCATION_ID IS NOT NULL +-- ClickHouse: Int column in MergeTree defaults to 0 when missing, not NULL +WHERE location_id != 0 diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_trips.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_trips.sql new file mode 100644 index 0000000..43158e6 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/models/staging/stg_trips.sql @@ -0,0 +1,106 @@ +{{ + config( + materialized = 'view', + schema = 'staging' + ) +}} +{# + Migration note: schema uses lowercase 'staging' vs Snowflake 'STAGING'. + ClickHouse identifiers are case-sensitive; dbt-clickhouse lowercases via + the generate_schema_name macro in macros/generate_schema_name.sql. +#} + +-- ════════════════════════════════════════════════════════════════════════════ +-- stg_trips — Staging layer: raw → typed, JSON flattened +-- +-- SNOWFLAKE → CLICKHOUSE TRANSLATION SUMMARY: +-- VARIANT colon-path → JSONExtract*() functions +-- DATEDIFF() → dateDiff() (lowercase 'd') +-- ::FLOAT / ::INTEGER → removed (JSONExtract returns typed value directly) +-- IS NOT NULL → != '' (ClickHouse MergeTree String cannot be NULL) +-- DROPOFF > PICKUP → same (DateTime comparison works identically) +-- schema = 'STAGING' → schema = 'staging' +-- ════════════════════════════════════════════════════════════════════════════ + +WITH source AS ( + SELECT * FROM {{ source('raw', 'trips_raw') }} FINAL + -- FINAL forces synchronous deduplication of trips_raw (ReplacingMergeTree(_synced_at)). + -- Without FINAL, duplicate trip_ids from migration retries or producer retries would + -- propagate into all downstream models (fact_trips, agg_hourly_zone_trips). + -- Migration note: Snowflake source was source('raw', 'TRIPS_RAW') (uppercase). + -- ClickHouse table names are case-sensitive; table was created with lowercase name. +), + +flattened AS ( + SELECT + trip_id, + vendor_id, + pickup_at, + dropoff_at, + + -- Snowflake: DATEDIFF('minute', PICKUP_DATETIME, DROPOFF_DATETIME) + -- ClickHouse: dateDiff() — lowercase 'd', same argument order + dateDiff('minute', pickup_at, dropoff_at) AS duration_minutes, + + passenger_count, + trip_distance_miles, + total_amount_usd, + tip_amount_usd, + fare_amount_usd, + extra_amount_usd, + mta_tax_usd, + tolls_amount_usd, + pickup_location_id, + dropoff_location_id, + payment_type_id, + rate_code_id, + store_fwd_flag, + ingested_at, + + -- ──────────────────────────────────────────────────────────────────── + -- VARIANT → JSON flattening + -- Snowflake used colon-path syntax + cast: TRIP_METADATA:driver.rating::FLOAT + -- ClickHouse uses JSONExtract functions; no cast needed — function returns typed value. + -- ──────────────────────────────────────────────────────────────────── + + -- Snowflake: TRIP_METADATA:driver.rating::FLOAT + JSONExtractFloat(trip_metadata, 'driver', 'rating') AS driver_rating, + + -- Snowflake: TRIP_METADATA:driver.trips_completed::INTEGER + JSONExtractInt(trip_metadata, 'driver', 'trips_completed') AS driver_trips_completed, + + -- Snowflake: TRIP_METADATA:driver.vehicle_type::VARCHAR + JSONExtractString(trip_metadata, 'driver', 'vehicle_type') AS vehicle_type, + + -- Snowflake: TRIP_METADATA:app.platform::VARCHAR + JSONExtractString(trip_metadata, 'app', 'platform') AS app_platform, + + -- Snowflake: TRIP_METADATA:app.version::VARCHAR + JSONExtractString(trip_metadata, 'app', 'version') AS app_version, + + -- Snowflake: TRIP_METADATA:app.surge_multiplier::FLOAT + JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier') AS surge_multiplier, + + -- Snowflake: TRIP_METADATA:route.estimated_minutes::INTEGER + JSONExtractInt(trip_metadata, 'route', 'estimated_minutes') AS route_estimated_minutes, + + -- Snowflake: TRIP_METADATA:route.actual_minutes::INTEGER + JSONExtractInt(trip_metadata, 'route', 'actual_minutes') AS route_actual_minutes, + + -- Snowflake: TRIP_METADATA:route.traffic_level::VARCHAR + JSONExtractString(trip_metadata, 'route', 'traffic_level') AS traffic_level + + FROM source + + -- Migration note: Snowflake used IS NOT NULL for nullable columns. + -- ClickHouse MergeTree String columns default to '' not NULL, so use != ''. + -- DateTime columns in ClickHouse cannot be NULL either — use > '1970-01-01'. + WHERE trip_id != '' + AND pickup_at > toDateTime('1970-01-01 00:00:00') + AND dropoff_at > toDateTime('1970-01-01 00:00:00') + -- Snowflake: AND DROPOFF_DATETIME > PICKUP_DATETIME + -- ClickHouse: same comparison — DateTime supports > operator identically + AND dropoff_at > pickup_at +) + +SELECT * FROM flattened diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/package-lock.yml b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/package-lock.yml new file mode 100644 index 0000000..f5ee93e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/package-lock.yml @@ -0,0 +1,5 @@ +packages: + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.3.3 +sha1_hash: dd1e1feb2d2bbce79e7a255cd309a60e6548df0b diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/packages.yml b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/packages.yml new file mode 100644 index 0000000..8965413 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/packages.yml @@ -0,0 +1,11 @@ +packages: + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<2.0.0"] + +# ────────────────────────────────────────────────────────────────────────────── +# Migration note: Part 1 (Snowflake) included dbt_expectations for column-value +# range tests. dbt_expectations is NOT compatible with the dbt-clickhouse adapter +# and has been intentionally omitted. Equivalent coverage is provided via: +# 1. Native dbt tests (not_null, unique, accepted_values) +# 2. Custom SQL tests in tests/ (assert_trips_not_future.sql, assert_revenue_positive.sql) +# ────────────────────────────────────────────────────────────────────────────── diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/profiles.yml.example b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/profiles.yml.example new file mode 100644 index 0000000..71f78f5 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/profiles.yml.example @@ -0,0 +1,27 @@ +# Required: pip install dbt-clickhouse>=1.7.0 +# The delete_insert incremental strategy and materialized_view materialization +# require dbt-clickhouse 1.7.0 or later. +nyc_taxi_ch: + target: dev + outputs: + dev: + type: clickhouse + # Migration note: Snowflake profile used account/warehouse/role/database. + # ClickHouse uses host/port/schema — no warehouse or role concepts. + # The schema here acts as both the database and default schema in ClickHouse. + schema: nyc_taxi_ch + host: "{{ env_var('CLICKHOUSE_HOST') }}" + port: 8443 # HTTPS native protocol port (ClickHouse Cloud default) + user: "{{ env_var('CLICKHOUSE_USER', 'default') }}" + password: "{{ env_var('CLICKHOUSE_PASSWORD') }}" + secure: true # TLS required for ClickHouse Cloud + # verify: false # uncomment if using self-signed cert in dev/local + + prod: + type: clickhouse + schema: nyc_taxi_ch + host: "{{ env_var('CLICKHOUSE_HOST_PROD') }}" + port: 8443 + user: "{{ env_var('CLICKHOUSE_USER_PROD', 'default') }}" + password: "{{ env_var('CLICKHOUSE_PASSWORD_PROD') }}" + secure: true diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_revenue_positive.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_revenue_positive.sql new file mode 100644 index 0000000..0135bd4 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_revenue_positive.sql @@ -0,0 +1,23 @@ +-- ════════════════════════════════════════════════════════════════════════════ +-- Custom test: assert_revenue_positive +-- Verifies all fare amounts are non-negative (no refunds or negative fares). +-- +-- A PASSING test returns 0 rows. Any rows returned = test failure. +-- +-- Migration note: In Snowflake Part 1, this was covered by dbt_expectations: +-- dbt_expectations.expect_column_values_to_be_between: +-- min_value: 0 +-- max_value: 1000 +-- dbt_expectations is not compatible with dbt-clickhouse, so this custom SQL +-- test replaces it. The max_value bound is intentionally omitted — unusually +-- high fares are valid data (e.g., long airport trips with surge pricing). +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT + trip_id, + total_amount_usd, + fare_amount_usd +FROM {{ ref('fact_trips') }} FINAL +-- FINAL forces synchronous deduplication — required for ReplacingMergeTree correctness +WHERE total_amount_usd < 0 + OR fare_amount_usd < 0 diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_trips_not_future.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_trips_not_future.sql new file mode 100644 index 0000000..2edd9d6 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/tests/assert_trips_not_future.sql @@ -0,0 +1,18 @@ +-- ════════════════════════════════════════════════════════════════════════════ +-- Custom test: assert_trips_not_future +-- Verifies no trip has a pickup_at timestamp in the future. +-- +-- A PASSING test returns 0 rows. Any rows returned = test failure. +-- +-- Migration note: Identical business logic to Part 1 (Snowflake). +-- ClickHouse dialect: now() replaces CURRENT_TIMESTAMP() — same semantics. +-- This test replaces Part 1's dbt_expectations range test on pickup_at. +-- ════════════════════════════════════════════════════════════════════════════ + +SELECT + trip_id, + pickup_at, + now() AS current_time +FROM {{ ref('fact_trips') }} FINAL +-- FINAL forces synchronous deduplication — required for ReplacingMergeTree correctness +WHERE pickup_at > now() diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/Dockerfile b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/Dockerfile new file mode 100644 index 0000000..8dc8838 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY producer.py . + +CMD ["python", "-u", "producer.py"] diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/producer.py b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/producer.py new file mode 100644 index 0000000..39e0111 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/producer.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +""" +NYC Taxi Trip Producer — ClickHouse +------------------------------------- +Continuously generates realistic fake taxi trips and inserts them into +default.trips_raw in ClickHouse Cloud, keeping the dashboards live after +producer cutover. + +Configuration (env vars): + CLICKHOUSE_HOST ClickHouse Cloud hostname (required) + CLICKHOUSE_PORT Default: 8443 + CLICKHOUSE_USER Default: default + CLICKHOUSE_PASSWORD ClickHouse password (required) + TRIPS_PER_MINUTE Default: 60 + BATCH_INTERVAL_SECONDS Default: 10 + LOG_LEVEL Default: INFO +""" +import json +import logging +import os +import random +import signal +import sys +import time +import uuid +from datetime import datetime, timezone, timedelta + +import clickhouse_connect + +# ── Configuration ────────────────────────────────────────────────────────────── + +CLICKHOUSE_HOST = os.environ["CLICKHOUSE_HOST"] +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT", "8443")) +CLICKHOUSE_USER = os.environ.get("CLICKHOUSE_USER", "default") +CLICKHOUSE_PASSWORD = os.environ["CLICKHOUSE_PASSWORD"] + +TRIPS_PER_MINUTE = float(os.environ.get("TRIPS_PER_MINUTE", "60")) +BATCH_INTERVAL_SECS = int(os.environ.get("BATCH_INTERVAL_SECONDS", "10")) +TRIPS_PER_BATCH = max(1, round(TRIPS_PER_MINUTE * BATCH_INTERVAL_SECS / 60)) + +logging.basicConfig( + level=getattr(logging, os.environ.get("LOG_LEVEL", "INFO").upper(), logging.INFO), + format="%(asctime)s [ch-producer] %(levelname)s %(message)s", + datefmt="%H:%M:%S", + stream=sys.stdout, +) +log = logging.getLogger(__name__) + +# ── Static reference data ────────────────────────────────────────────────────── + +# Manhattan zones appear 4x more often (matches historical distribution) +_MANHATTAN = ( + list(range(4, 12)) + list(range(13, 45)) + list(range(46, 78)) + + list(range(79, 104)) + [107, 113, 114, 125, 140, 141, 142, 143, 144, 148, + 151, 152, 153, 158, 161, 162, 163, 164, 166, 170, 186, 194, 202, 209, + 211, 224, 231, 234, 236, 239, 243, 244, 246, 249, 261, 262, 263] +) +_OTHER = [z for z in range(1, 266) if z not in _MANHATTAN] +WEIGHTED_ZONES = _MANHATTAN * 4 + _OTHER + +VEHICLE_TYPES = ["Sedan", "SUV", "Minivan", "Luxury"] +PLATFORMS = ["iOS", "Android", "Web"] +TRAFFIC_LEVELS = ["none", "light", "moderate", "heavy"] +TRAFFIC_WEIGHTS = [20, 40, 30, 10] + +APP_VERSIONS = [ + f"{maj}.{minor}.{patch}" + for maj in range(2, 5) + for minor in range(0, 8) + for patch in range(0, 5) +] + +# ── Column list for default.trips_raw ───────────────────────────────────────── + +COLUMNS = [ + "trip_id", "vendor_id", "pickup_at", "dropoff_at", + "passenger_count", "trip_distance_miles", "rate_code_id", "store_fwd_flag", + "pickup_location_id", "dropoff_location_id", "payment_type_id", + "fare_amount_usd", "extra_amount_usd", "mta_tax_usd", + "tip_amount_usd", "tolls_amount_usd", "total_amount_usd", + "ingested_at", "trip_metadata", +] + +# ── Trip generation ──────────────────────────────────────────────────────────── + +def make_trip() -> list: + """Return one row list matching the COLUMNS order above.""" + vendor_id = random.randint(1, 3) + pu_zone = random.choice(WEIGHTED_ZONES) + do_zone = random.choice(WEIGHTED_ZONES) + passenger_count = random.choices([1, 2, 3, 4, 5, 6], weights=[55, 20, 10, 8, 5, 2])[0] + payment_type = random.choices([1, 2, 3, 4, 5, 6], weights=[65, 30, 1, 2, 1, 1])[0] + rate_code_id = random.choices([1, 2, 3, 4, 5, 6], weights=[88, 4, 1, 1, 4, 2])[0] + store_fwd = random.choices(["N", "Y"], weights=[98, 2])[0] + + distance = round(max(0.1, min(random.expovariate(1 / 4.0), 60.0)), 2) + + traffic_level = random.choices(TRAFFIC_LEVELS, weights=TRAFFIC_WEIGHTS)[0] + traffic_factor = {"none": 3.5, "light": 4.5, "moderate": 6.0, "heavy": 8.5}[traffic_level] + duration_min = max(3, min(120, int(distance * traffic_factor + random.gauss(3, 2)))) + estimated_min = max(3, int(distance * 4.5 + random.gauss(2, 1))) + + dropoff_dt = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(seconds=random.randint(0, 90)) + pickup_dt = dropoff_dt - timedelta(minutes=duration_min) + + fare = round(3.00 + distance * 1.75, 2) + extra = round(random.choices([0.0, 0.5, 1.0], weights=[60, 30, 10])[0], 2) + mta = 0.50 + tolls = round(random.choices([0.0, 6.12, 11.52], weights=[85, 10, 5])[0], 2) + tip = round(fare * random.uniform(0.15, 0.25), 2) if payment_type == 1 else 0.0 + total = round(fare + extra + mta + tip + tolls, 2) + + surge_roll = random.random() + if surge_roll < 0.03: + surge = round(random.uniform(2.0, 3.5), 1) + elif surge_roll < 0.10: + surge = round(random.uniform(1.5, 2.0), 1) + elif surge_roll < 0.25: + surge = round(random.uniform(1.1, 1.5), 1) + else: + surge = 1.0 + + metadata = json.dumps({ + "driver": { + "rating": max(1.0, min(5.0, round(random.gauss(4.6, 0.3), 1))), + "trips_completed": random.randint(50, 8000), + "vehicle_type": random.choice(VEHICLE_TYPES), + }, + "app": { + "version": random.choice(APP_VERSIONS), + "platform": random.choice(PLATFORMS), + "surge_multiplier": surge, + }, + "route": { + "estimated_minutes": estimated_min, + "actual_minutes": duration_min, + "traffic_level": traffic_level, + }, + }) + + now = datetime.now(timezone.utc).replace(tzinfo=None) + + return [ + str(uuid.uuid4()), # trip_id + vendor_id, # vendor_id + pickup_dt, # pickup_at + dropoff_dt, # dropoff_at + passenger_count, # passenger_count + distance, # trip_distance_miles + rate_code_id, # rate_code_id + store_fwd, # store_fwd_flag + pu_zone, # pickup_location_id + do_zone, # dropoff_location_id + payment_type, # payment_type_id + fare, # fare_amount_usd + extra, # extra_amount_usd + mta, # mta_tax_usd + tip, # tip_amount_usd + tolls, # tolls_amount_usd + total, # total_amount_usd + now, # ingested_at + metadata, # trip_metadata + ] + +# ── ClickHouse connection ────────────────────────────────────────────────────── + +def connect() -> clickhouse_connect.driver.Client: + log.info("Connecting host=%s port=%d user=%s", CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER) + client = clickhouse_connect.get_client( + host = CLICKHOUSE_HOST, + port = CLICKHOUSE_PORT, + username = CLICKHOUSE_USER, + password = CLICKHOUSE_PASSWORD, + secure = True, + ) + # Verify connectivity + client.command("SELECT 1") + log.info("Connected.") + return client + +# ── Main loop ────────────────────────────────────────────────────────────────── + +def run(): + client = connect() + total = 0 + start = time.monotonic() + + log.info("Producer running — %.0f trips/min, %d per batch, interval %ds", + TRIPS_PER_MINUTE, TRIPS_PER_BATCH, BATCH_INTERVAL_SECS) + + while True: + batch_start = time.monotonic() + rows = [make_trip() for _ in range(TRIPS_PER_BATCH)] + + try: + client.insert("default.trips_raw", rows, column_names=COLUMNS) + total += len(rows) + elapsed_min = (time.monotonic() - start) / 60 or 0.001 + log.info("✓ inserted %3d trips | total=%6d | actual rate=%.1f trips/min", + len(rows), total, total / elapsed_min) + except Exception as exc: + log.error("Insert failed: %s — reconnecting in 5s", exc) + time.sleep(5) + try: + client = connect() + except Exception as conn_exc: + log.error("Reconnect failed: %s — will retry next batch", conn_exc) + continue + + sleep_secs = max(0.0, BATCH_INTERVAL_SECS - (time.monotonic() - batch_start)) + time.sleep(sleep_secs) + + +def _shutdown(sig, _frame): + log.info("Received signal %s — shutting down.", sig) + sys.exit(0) + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, _shutdown) + signal.signal(signal.SIGINT, _shutdown) + + log.info("NYC Taxi Trip Producer (ClickHouse) starting " + "(TRIPS_PER_MINUTE=%.0f BATCH_INTERVAL=%ds TRIPS_PER_BATCH=%d)", + TRIPS_PER_MINUTE, BATCH_INTERVAL_SECS, TRIPS_PER_BATCH) + + backoff = 15 + while True: + try: + run() + except KeyboardInterrupt: + log.info("Interrupted.") + sys.exit(0) + except Exception as exc: + log.error("Unexpected error: %s — retrying in %ds", exc, backoff) + time.sleep(backoff) + backoff = min(backoff * 2, 120) + else: + backoff = 15 diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/requirements.txt b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/requirements.txt new file mode 100644 index 0000000..15946ea --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/producer/requirements.txt @@ -0,0 +1 @@ +clickhouse-connect>=0.7.0,<1.0 diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/00_seed_zones.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/00_seed_zones.sql new file mode 100644 index 0000000..012baef --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/00_seed_zones.sql @@ -0,0 +1,65 @@ +-- ============================================================ +-- Script 00: Seed taxi_zones in ClickHouse +-- Run before: dbt run (section 7.1) +-- +-- Creates and populates default.taxi_zones with 265 synthetic +-- NYC TLC zone records — identical distribution to the Snowflake +-- seed in 01-setup-snowflake/scripts/02_seed_data.sql. +-- +-- The zone data is static reference data (265 rows). It is seeded +-- once via this script before the first dbt run. +-- stg_taxi_zones reads from this table via source('raw', 'taxi_zones'). +-- Named taxi_zones (not dim_taxi_zones) to avoid collision with analytics.dim_taxi_zones. +-- ============================================================ + +CREATE TABLE IF NOT EXISTS default.taxi_zones ( + location_id UInt16, + borough String, + zone String, + service_zone String +) +ENGINE = MergeTree() +ORDER BY (location_id); + +TRUNCATE TABLE default.taxi_zones; + +INSERT INTO default.taxi_zones (location_id, borough, zone, service_zone) +SELECT + n AS location_id, + CASE + WHEN n BETWEEN 1 AND 69 THEN 'Manhattan' + WHEN n BETWEEN 70 AND 139 THEN 'Brooklyn' + WHEN n BETWEEN 140 AND 199 THEN 'Queens' + WHEN n BETWEEN 200 AND 235 THEN 'Bronx' + WHEN n BETWEEN 236 AND 250 THEN 'Staten Island' + ELSE 'EWR' + END AS borough, + concat( + CASE (n % 20) + WHEN 0 THEN 'Airport' WHEN 1 THEN 'Heights' + WHEN 2 THEN 'Gardens' WHEN 3 THEN 'Park' + WHEN 4 THEN 'Hill' WHEN 5 THEN 'Village' + WHEN 6 THEN 'Square' WHEN 7 THEN 'Bridge' + WHEN 8 THEN 'Terrace' WHEN 9 THEN 'Point' + WHEN 10 THEN 'Flats' WHEN 11 THEN 'Beach' + WHEN 12 THEN 'Junction' WHEN 13 THEN 'Manor' + WHEN 14 THEN 'Harbor' WHEN 15 THEN 'Estates' + WHEN 16 THEN 'Commons' WHEN 17 THEN 'District' + WHEN 18 THEN 'Place' ELSE 'Center' + END, + ' ', + toString(n) + ) AS zone, + CASE (n % 3) + WHEN 0 THEN 'Yellow Zone' + WHEN 1 THEN 'Boro Zone' + ELSE 'Airports' + END AS service_zone +FROM ( + SELECT number + 1 AS n + FROM numbers(265) +); + +-- Verify +SELECT count() AS zone_count FROM default.taxi_zones; +-- Expected: 265 diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/01_verify_migration.sh b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/01_verify_migration.sh new file mode 100755 index 0000000..db991c5 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/01_verify_migration.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# ============================================================ +# 01_verify_migration.sh +# Verifies row count parity between Snowflake and ClickHouse. +# +# Exit 0 → counts match (safe to proceed) +# Exit 1 → count is zero, unexpected, or parity check fails +# +# Standalone usage: +# source .env && source .clickhouse_state +# bash scripts/01_verify_migration.sh +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "${SCRIPT_DIR}")" + +# Source environment files +if [[ -f "${LAB_DIR}/.env" ]]; then + set -a; source "${LAB_DIR}/.env"; set +a +fi +if [[ -f "${LAB_DIR}/.clickhouse_state" ]]; then + set -a; source "${LAB_DIR}/.clickhouse_state"; set +a +fi + +# --------------------------------------------------------------------------- +# Colors & log helpers +# --------------------------------------------------------------------------- +BOLD="\033[1m"; RESET="\033[0m" +GREEN="\033[1;32m"; RED="\033[1;31m"; YELLOW="\033[1;33m"; BLUE="\033[1;34m" + +info() { echo -e "${BLUE} ${*}${RESET}"; } +ok() { echo -e "${GREEN} ✓ ${*}${RESET}"; } +warn() { echo -e "${YELLOW} ⚠ ${*}${RESET}"; } +die() { echo -e "${RED}${BOLD} ✗ ${*}${RESET}" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Validate required vars +# --------------------------------------------------------------------------- +: "${CLICKHOUSE_HOST:?CLICKHOUSE_HOST is not set. Source .clickhouse_state first.}" +: "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is not set.}" +CLICKHOUSE_PORT="${CLICKHOUSE_PORT:-8443}" +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" + +# --------------------------------------------------------------------------- +# Query helpers +# --------------------------------------------------------------------------- +clickhouse_query() { + local query="$1" + local response http_code body + response=$(curl -s -w "\n%{http_code}" \ + "https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/" \ + -u "${CLICKHOUSE_USER}:${CLICKHOUSE_PASSWORD}" \ + --data-binary "${query}" 2>&1) + http_code=$(echo "${response}" | tail -1) + body=$(echo "${response}" | sed '$d') + if [[ "${http_code}" != "200" ]]; then + die "ClickHouse query failed (HTTP ${http_code}): ${body}" + fi + echo "${body}" +} + +snowflake_count() { + python3 - <<'PYEOF' +import os, sys +try: + import snowflake.connector +except ImportError: + print("SKIP") + sys.exit(0) + +required = ["SNOWFLAKE_ORG", "SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_PASSWORD"] +missing = [v for v in required if not os.environ.get(v)] +if missing: + print("SKIP") + sys.exit(0) + +try: + conn = snowflake.connector.connect( + account = f"{os.environ['SNOWFLAKE_ORG']}-{os.environ['SNOWFLAKE_ACCOUNT']}", + user = os.environ["SNOWFLAKE_USER"], + password = os.environ["SNOWFLAKE_PASSWORD"], + warehouse = "TRANSFORM_WH", + database = "NYC_TAXI_DB", + schema = "RAW", + login_timeout = 15, + ) + cur = conn.cursor() + cur.execute("SELECT COUNT(*) FROM NYC_TAXI_DB.RAW.TRIPS_RAW") + print(cur.fetchone()[0]) + cur.close() + conn.close() +except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + print("SKIP") +PYEOF +} + +fmt_number() { + python3 -c "print(f'{int(\"$1\"):,}')" 2>/dev/null || echo "$1" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +DIVIDER="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +echo "" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo -e "${BOLD} Migration Parity Check${RESET}" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" + +# 1. ClickHouse row count +info "Querying ClickHouse default.trips_raw row count…" +CH_COUNT_RAW=$(clickhouse_query "SELECT count() FROM default.trips_raw") +CH_COUNT=$(echo "${CH_COUNT_RAW}" | tr -d '[:space:]') +if ! [[ "${CH_COUNT}" =~ ^[0-9]+$ ]]; then + die "Could not retrieve ClickHouse row count (got: '${CH_COUNT}'). Check CLICKHOUSE_HOST / credentials." +fi + +CH_FMT=$(fmt_number "${CH_COUNT}") +ok "ClickHouse default.trips_raw: ${BOLD}${CH_FMT}${RESET} rows" + +if [[ "${CH_COUNT}" -eq 0 ]]; then + echo "" + echo -e "${RED}${BOLD} ✗ ClickHouse trips_raw is empty.${RESET}" + echo " Run the migration script first:" + echo " python scripts/02_migrate_trips.py" + echo -e "${BOLD}${DIVIDER}${RESET}" + exit 1 +fi + +# 2. Snowflake row count +echo "" +info "Querying Snowflake NYC_TAXI_DB.RAW.TRIPS_RAW row count…" +SF_RESULT=$(snowflake_count) + +if [[ "${SF_RESULT}" == "SKIP" ]]; then + warn "Snowflake credentials not available — skipping automated parity check." + warn "Run manually: SELECT COUNT(*) FROM NYC_TAXI_DB.RAW.TRIPS_RAW;" + warn "Compare against ClickHouse count: ${CH_FMT}" +elif [[ "${SF_RESULT}" =~ ^[0-9]+$ ]]; then + SF_FMT=$(fmt_number "${SF_RESULT}") + ok "Snowflake NYC_TAXI_DB.RAW.TRIPS_RAW: ${BOLD}${SF_FMT}${RESET} rows" + + # Parity check — direction-aware: + # CH >= SF → expected post-cutover (live producer adds rows); always PASS + # CH < SF → missing rows; PASS only if gap <= 0.01% (in-flight during migration) + MISSING=$(( SF_RESULT - CH_COUNT )) # positive = CH is behind; negative = CH is ahead + + echo "" + if [[ "${MISSING}" -le 0 ]]; then + EXTRA=$(( -MISSING )) + EXTRA_FMT=$(fmt_number "${EXTRA}") + ok "Row count parity: PASS (ClickHouse has ${EXTRA_FMT} extra rows — live producer is running)" + else + PCT=$(python3 -c "print(f'{${MISSING} / ${SF_RESULT} * 100:.4f}')" 2>/dev/null || echo "?") + MISSING_FMT=$(fmt_number "${MISSING}") + if python3 -c "import sys; sys.exit(0 if ${MISSING} / ${SF_RESULT} * 100 <= 0.01 else 1)" 2>/dev/null; then + ok "Row count parity: PASS (ClickHouse is ${MISSING_FMT} rows behind = ${PCT}% — within 0.01% threshold)" + else + warn "ClickHouse is missing ${MISSING_FMT} rows (${PCT}%) — exceeds 0.01% threshold" + warn "Re-run with --resume to migrate the gap:" + warn " python scripts/02_migrate_trips.py --resume" + fi + fi +else + warn "Could not retrieve Snowflake count — skipping parity check." +fi + +echo "" + +# 3. Spot check: trip_metadata populated +info "Spot check: trip_metadata column…" +META_COUNT_RAW=$(clickhouse_query "SELECT countIf(trip_metadata != '') FROM default.trips_raw") +META_COUNT=$(echo "${META_COUNT_RAW}" | tr -d '[:space:]') +META_FMT=$(fmt_number "${META_COUNT:-0}") + +if [[ "${META_COUNT}" =~ ^[0-9]+$ ]] && [[ "${META_COUNT}" -gt 0 ]]; then + ok "trip_metadata populated: ${META_FMT} non-empty rows" +else + warn "trip_metadata appears empty. This may indicate a column mapping issue in the migration script." +fi + +# 4. Spot check: pickup_at range +info "Spot check: pickup_at date range…" +RANGE_RAW=$(clickhouse_query "SELECT min(toDate(pickup_at)), max(toDate(pickup_at)) FROM default.trips_raw") +echo -e " pickup_at range: ${BOLD}${RANGE_RAW}${RESET}" + +# 5. Final verdict +echo "" +echo -e "${GREEN}${BOLD} ✓ ClickHouse has ${CH_FMT} rows — migration looks complete${RESET}" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_migrate_trips.py b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_migrate_trips.py new file mode 100755 index 0000000..8dabb85 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_migrate_trips.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +02_migrate_trips.py — Batch migrate NYC Taxi trips from Snowflake to ClickHouse. + +Usage: + source .env && source .clickhouse_state + python scripts/02_migrate_trips.py + python scripts/02_migrate_trips.py --batch-size 50000 + python scripts/02_migrate_trips.py --resume # skip rows already in ClickHouse + +Reads from: NYC_TAXI_DB.RAW.TRIPS_RAW (Snowflake) +Writes to: default.trips_raw (ClickHouse) + +Required environment variables: + SNOWFLAKE_ORG e.g. myorg + SNOWFLAKE_ACCOUNT e.g. abc12345 + SNOWFLAKE_USER e.g. MIGRATION_USER + SNOWFLAKE_PASSWORD + CLICKHOUSE_HOST e.g. abc123.us-east-1.aws.clickhouse.cloud + CLICKHOUSE_PASSWORD + CLICKHOUSE_USER (default: default) + CLICKHOUSE_PORT (default: 8443) + +Install dependencies: + pip install snowflake-connector-python clickhouse-connect +""" + +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone + +# --------------------------------------------------------------------------- +# Dependency check +# --------------------------------------------------------------------------- +try: + import snowflake.connector +except ImportError: + sys.exit("Missing dependency: pip install snowflake-connector-python") + +try: + import clickhouse_connect +except ImportError: + sys.exit("Missing dependency: pip install clickhouse-connect") + +# --------------------------------------------------------------------------- +# Column mapping: Snowflake name → ClickHouse name +# Order must match the SELECT below. +# --------------------------------------------------------------------------- +COLUMN_MAP = [ + ("TRIP_ID", "trip_id"), + ("VENDOR_ID", "vendor_id"), + ("PICKUP_DATETIME", "pickup_at"), + ("DROPOFF_DATETIME", "dropoff_at"), + ("PASSENGER_COUNT", "passenger_count"), + ("TRIP_DISTANCE", "trip_distance_miles"), + ("RATECODE_ID", "rate_code_id"), + ("STORE_FWD_FLAG", "store_fwd_flag"), + ("PU_LOCATION_ID", "pickup_location_id"), + ("DO_LOCATION_ID", "dropoff_location_id"), + ("PAYMENT_TYPE", "payment_type_id"), + ("FARE_AMOUNT", "fare_amount_usd"), + ("EXTRA", "extra_amount_usd"), + ("MTA_TAX", "mta_tax_usd"), + ("TIP_AMOUNT", "tip_amount_usd"), + ("TOLLS_AMOUNT", "tolls_amount_usd"), + ("TOTAL_AMOUNT", "total_amount_usd"), + ("INGESTED_AT", "ingested_at"), + ("TRIP_METADATA", "trip_metadata"), +] + +SF_COLUMNS = [sf for sf, _ in COLUMN_MAP] +CH_COLUMNS = [ch for _, ch in COLUMN_MAP] + +# Index of TRIP_METADATA in the row tuple (last column) +METADATA_IDX = SF_COLUMNS.index("TRIP_METADATA") + + +def get_env(name: str, default: str = None) -> str: + val = os.environ.get(name, default) + if val is None: + sys.exit(f"Error: environment variable {name} is not set. Source .env and .clickhouse_state first.") + return val + + +def connect_snowflake() -> snowflake.connector.SnowflakeConnection: + org = get_env("SNOWFLAKE_ORG") + account = get_env("SNOWFLAKE_ACCOUNT") + user = get_env("SNOWFLAKE_USER") + password = get_env("SNOWFLAKE_PASSWORD") + account_id = f"{org}-{account}" + print(f" Connecting to Snowflake: {account_id}") + conn = snowflake.connector.connect( + account=account_id, + user=user, + password=password, + warehouse="TRANSFORM_WH", + database="NYC_TAXI_DB", + schema="RAW", + ) + print(" Snowflake: connected") + return conn + + +def connect_clickhouse(): + host = get_env("CLICKHOUSE_HOST") + password = get_env("CLICKHOUSE_PASSWORD") + user = get_env("CLICKHOUSE_USER", "default") + port = int(get_env("CLICKHOUSE_PORT", "8443")) + print(f" Connecting to ClickHouse: {host}:{port}") + client = clickhouse_connect.get_client( + host=host, + port=port, + username=user, + password=password, + secure=True, + ) + print(" ClickHouse: connected") + return client + + +def get_resume_watermark(ch_client) -> datetime | None: + """Return the max pickup_at already in ClickHouse, or None if table is empty.""" + result = ch_client.query("SELECT max(pickup_at) FROM default.trips_raw") + val = result.first_row[0] + if val is None or (hasattr(val, 'year') and val.year == 1970): + return None + return val + + +def build_sf_query(resume_after: datetime | None, batch_size: int) -> str: + select = ", ".join(SF_COLUMNS) + base = f"SELECT {select} FROM NYC_TAXI_DB.RAW.TRIPS_RAW" + if resume_after: + ts = resume_after.strftime("%Y-%m-%d %H:%M:%S.%f") + base += f" WHERE PICKUP_DATETIME > '{ts}'" + base += " ORDER BY PICKUP_DATETIME, TRIP_ID" + return base + + +def coerce_row(row: tuple) -> list: + """Convert Snowflake row values to ClickHouse-compatible types.""" + row = list(row) + + # TRIP_METADATA: Snowflake VARIANT comes back as a dict or already-serialized string. + meta = row[METADATA_IDX] + if meta is None: + row[METADATA_IDX] = "" + elif isinstance(meta, dict): + row[METADATA_IDX] = json.dumps(meta) + else: + row[METADATA_IDX] = str(meta) + + # STORE_FWD_FLAG: coerce None → "" + sfwd_idx = SF_COLUMNS.index("STORE_FWD_FLAG") + if row[sfwd_idx] is None: + row[sfwd_idx] = "" + + # Numeric NULLs → 0 + numeric_cols = [ + "VENDOR_ID", "PASSENGER_COUNT", "TRIP_DISTANCE", "RATECODE_ID", + "PU_LOCATION_ID", "DO_LOCATION_ID", "PAYMENT_TYPE", + "FARE_AMOUNT", "EXTRA", "MTA_TAX", "TIP_AMOUNT", "TOLLS_AMOUNT", "TOTAL_AMOUNT", + ] + for col in numeric_cols: + idx = SF_COLUMNS.index(col) + if row[idx] is None: + row[idx] = 0 + + return row + + +def fmt(n: int) -> str: + return f"{n:,}" + + +def eta_str(elapsed: float, done: int, total: int) -> str: + if done == 0: + return "calculating..." + rate = done / elapsed + remaining_rows = total - done + remaining_secs = remaining_rows / rate + m, s = divmod(int(remaining_secs), 60) + h, m = divmod(m, 60) + if h > 0: + return f"{h}h {m}m remaining" + elif m > 0: + return f"{m}m {s}s remaining" + else: + return f"{s}s remaining" + + +def main(): + parser = argparse.ArgumentParser(description="Migrate NYC Taxi trips from Snowflake to ClickHouse") + parser.add_argument("--batch-size", type=int, default=100_000, help="Rows per ClickHouse INSERT (default: 100000)") + parser.add_argument("--resume", action="store_true", help="Skip rows already in ClickHouse (uses max pickup_at as watermark)") + args = parser.parse_args() + + DIVIDER = "━" * 60 + print() + print(DIVIDER) + print(" NYC Taxi Migration: Snowflake → ClickHouse") + print(DIVIDER) + print() + + # Connect + sf_conn = connect_snowflake() + ch_client = connect_clickhouse() + print() + + # Resume watermark + resume_after = None + if args.resume: + resume_after = get_resume_watermark(ch_client) + if resume_after: + print(f" Resume mode: skipping rows with pickup_at <= {resume_after}") + else: + print(" Resume mode: ClickHouse table is empty, starting from the beginning") + print() + + # Total row count for progress + count_query = "SELECT COUNT(*) FROM NYC_TAXI_DB.RAW.TRIPS_RAW" + if resume_after: + ts = resume_after.strftime("%Y-%m-%d %H:%M:%S.%f") + count_query += f" WHERE PICKUP_DATETIME > '{ts}'" + sf_cur = sf_conn.cursor() + sf_cur.execute(count_query) + total_rows = sf_cur.fetchone()[0] + sf_cur.close() + print(f" Rows to migrate: {fmt(total_rows)}") + print(f" Batch size: {fmt(args.batch_size)}") + print() + + if total_rows == 0: + print(" Nothing to migrate. Exiting.") + sf_conn.close() + ch_client.close() + return + + # Stream rows from Snowflake in batches + query = build_sf_query(resume_after, args.batch_size) + sf_cur = sf_conn.cursor() + sf_cur.execute(query) + + total_inserted = 0 + batch: list[list] = [] + start_time = time.time() + + print(f" {'Rows inserted':<20} {'Elapsed':<12} {'ETA':<22} {'Rate'}") + print(f" {'-'*20} {'-'*12} {'-'*22} {'-'*15}") + + for raw_row in sf_cur: + batch.append(coerce_row(raw_row)) + + if len(batch) >= args.batch_size: + ch_client.insert("default.trips_raw", batch, column_names=CH_COLUMNS) + total_inserted += len(batch) + batch = [] + + elapsed = time.time() - start_time + rate = total_inserted / elapsed + eta = eta_str(elapsed, total_inserted, total_rows) + pct = total_inserted * 100 / total_rows + print( + f" {fmt(total_inserted):<20} " + f"{int(elapsed//60)}m {int(elapsed%60):02d}s " + f"{eta:<22} " + f"{fmt(int(rate))} rows/s ({pct:.1f}%)" + ) + + # Flush remainder + if batch: + ch_client.insert("default.trips_raw", batch, column_names=CH_COLUMNS) + total_inserted += len(batch) + + sf_cur.close() + sf_conn.close() + ch_client.close() + + elapsed = time.time() - start_time + rate = total_inserted / elapsed if elapsed > 0 else 0 + + print() + print(DIVIDER) + print(f" Migration complete") + print(f" Rows inserted: {fmt(total_inserted)}") + print(f" Total time: {int(elapsed//60)}m {int(elapsed%60):02d}s") + print(f" Avg rate: {fmt(int(rate))} rows/s") + print(DIVIDER) + print() + print(" Next step: verify with bash scripts/01_verify_migration.sh") + print() + + +if __name__ == "__main__": + main() diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_validate_parity.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_validate_parity.sql new file mode 100644 index 0000000..b56e3e1 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/02_validate_parity.sql @@ -0,0 +1,50 @@ +-- ============================================================ +-- 02_validate_parity.sql +-- Manual parity validation queries for ClickHouse SQL console. +-- Run these after the Python migration script to confirm data integrity. +-- ============================================================ +-- Connect to your ClickHouse Cloud service before running these queries. +-- First-run step: USE nyc_taxi_ch; + +-- 1. Row count comparison (run both and compare) +-- In ClickHouse SQL console: +SELECT 'trips_raw' AS table_name, count() AS row_count +FROM nyc_taxi_ch.trips_raw; + +-- In Snowflake (for reference): +-- SELECT COUNT(*) FROM NYC_TAXI_DB.RAW.TRIPS_RAW; + +-- 2. Date range check — should span 2019–2022 +SELECT + min(toDate(pickup_at)) AS earliest_trip, + max(toDate(pickup_at)) AS latest_trip, + count() AS total_rows +FROM nyc_taxi_ch.trips_raw; + +-- 3. trip_metadata JSON populated? +SELECT + countIf(trip_metadata != '') AS with_metadata, + countIf(trip_metadata = '') AS empty_metadata, + count() AS total +FROM nyc_taxi_ch.trips_raw; + +-- 4. Driver rating spot check (should be ~1.0–5.0) +SELECT + round(JSONExtractFloat(trip_metadata, 'driver', 'rating'), 1) AS rating, + count() AS trips +FROM nyc_taxi_ch.trips_raw +WHERE JSONExtractFloat(trip_metadata, 'driver', 'rating') > 0 +GROUP BY rating +ORDER BY rating; + +-- 5. Borough distribution (after dbt run — fact_trips) +SELECT pickup_borough, count() AS trips +FROM analytics.fact_trips FINAL +GROUP BY pickup_borough +ORDER BY trips DESC; + +-- OPTIONAL: Run only after CDC connector is active (Step 6 in setup.sh) +-- 6. CDC stream health — new trips appearing? +-- Run twice 5 minutes apart. The count should increase if CDC is running. +SELECT count() AS total_trips, max(ingested_at) AS latest_ingest +FROM nyc_taxi_ch.trips_raw; diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/03_cutover.sh b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/03_cutover.sh new file mode 100755 index 0000000..9ad8d70 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/03_cutover.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +# ============================================================ +# 03_cutover.sh +# Producer cutover: Snowflake → ClickHouse +# +# Steps: +# 1. Confirm intent (type "cutover") +# 2. Stop Snowflake trip producer +# 3. Run final dbt refresh on ClickHouse +# 4. Start ClickHouse producer +# 5. Verify new trips appear in ClickHouse +# +# Standalone usage: +# source .env && source .clickhouse_state +# bash scripts/03_cutover.sh +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "${SCRIPT_DIR}")" + +# Source environment files +if [[ -f "${LAB_DIR}/.env" ]]; then + set -a; source "${LAB_DIR}/.env"; set +a +fi +if [[ -f "${LAB_DIR}/.clickhouse_state" ]]; then + set -a; source "${LAB_DIR}/.clickhouse_state"; set +a +fi + +# --------------------------------------------------------------------------- +# Colors & log helpers +# --------------------------------------------------------------------------- +BOLD="\033[1m"; RESET="\033[0m" +GREEN="\033[1;32m"; RED="\033[1;31m"; YELLOW="\033[1;33m"; BLUE="\033[1;34m" + +info() { echo -e "${BLUE} ${*}${RESET}"; } +ok() { echo -e "${GREEN} ✓ ${*}${RESET}"; } +warn() { echo -e "${YELLOW} ⚠ ${*}${RESET}"; } +die() { echo -e "${RED}${BOLD} ✗ ${*}${RESET}" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Validate required vars +# --------------------------------------------------------------------------- +: "${CLICKHOUSE_HOST:?CLICKHOUSE_HOST is not set. Source .clickhouse_state first.}" +: "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is not set.}" +CLICKHOUSE_PORT="${CLICKHOUSE_PORT:-8443}" +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" + +DBT_DIR="${DBT_DIR:-${LAB_DIR}/dbt/nyc_taxi_dbt_ch}" + +# --------------------------------------------------------------------------- +# Query helper +# --------------------------------------------------------------------------- +clickhouse_query() { + local query="$1" + curl -s --fail \ + "https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/" \ + -u "${CLICKHOUSE_USER}:${CLICKHOUSE_PASSWORD}" \ + --data-binary "${query}" +} + +DIVIDER="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# --------------------------------------------------------------------------- +# Step 0: Confirmation +# --------------------------------------------------------------------------- +echo "" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo -e "${BOLD} NYC Taxi Lab — Producer Cutover: Snowflake → ClickHouse${RESET}" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" +echo -e "${YELLOW}${BOLD} WARNING: This will stop the Snowflake producer and redirect${RESET}" +echo -e "${YELLOW}${BOLD} live trip writes to ClickHouse Cloud.${RESET}" +echo "" +echo -e " To roll back after cutover:" +echo " 1. docker stop nyc_taxi_ch_producer" +echo " 2. docker start nyc_taxi_producer (restarts Snowflake producer)" +echo "" +printf " Type %b\"cutover\"%b to confirm, or Ctrl-C to abort: " "${BOLD}" "${RESET}" +read -r CONFIRM + +if [[ "${CONFIRM}" != "cutover" ]]; then + echo "" + warn "Confirmation not received. Aborting." + exit 0 +fi + +echo "" + +# --------------------------------------------------------------------------- +# Step 1: Stop Snowflake producer +# --------------------------------------------------------------------------- +echo -e "${BLUE}${BOLD}[1/3] Stopping Snowflake trip producer…${RESET}" +if docker stop nyc_taxi_producer 2>/dev/null; then + ok "nyc_taxi_producer stopped." +else + warn "nyc_taxi_producer was not running (may already be stopped). Continuing." +fi +echo "" + +# --------------------------------------------------------------------------- +# Step 2: Final dbt run on ClickHouse +# --------------------------------------------------------------------------- +echo -e "${BLUE}${BOLD}[2/3] Running final dbt refresh on ClickHouse…${RESET}" + +if [[ ! -d "${DBT_DIR}" ]]; then + warn "dbt directory not found at ${DBT_DIR} — skipping dbt run." + warn "Run manually: cd dbt && dbt run --profiles-dir dbt" +else + if ! command -v dbt >/dev/null 2>&1; then + warn "dbt not found in PATH — skipping dbt run." + warn "Run manually: cd ${DBT_DIR} && dbt run --profiles-dir \"${DBT_DIR}\"" + else + info "Running: dbt run --profiles-dir \"${DBT_DIR}\"" + if (cd "${DBT_DIR}" && dbt run --profiles-dir "${DBT_DIR}"); then + ok "dbt run completed successfully." + else + warn "dbt run exited with errors. Check output above." + warn "You may need to re-run: cd ${DBT_DIR} && dbt run --profiles-dir \"${DBT_DIR}\"" + fi + fi +fi +echo "" + +# --------------------------------------------------------------------------- +# Step 4: Start ClickHouse producer +# --------------------------------------------------------------------------- +echo -e "${BLUE}${BOLD}[3/3] Starting ClickHouse trip producer…${RESET}" + +# Build the ClickHouse producer image if not already present +PRODUCER_DIR="${LAB_DIR}/producer" +if ! docker image inspect nyc_taxi_ch_producer:latest >/dev/null 2>&1; then + info "Building nyc_taxi_ch_producer image from ${PRODUCER_DIR}…" + if ! docker build -t nyc_taxi_ch_producer:latest "${PRODUCER_DIR}"; then + die "Docker build failed. Check ${PRODUCER_DIR}/Dockerfile and ensure Docker is running." + fi + ok "Image built: nyc_taxi_ch_producer:latest" +else + info "Using existing nyc_taxi_ch_producer:latest image." +fi + +# Remove any existing stopped container with the same name +if docker inspect nyc_taxi_ch_producer >/dev/null 2>&1; then + info "Removing existing nyc_taxi_ch_producer container…" + docker rm -f nyc_taxi_ch_producer >/dev/null 2>&1 || true +fi + +# Write credentials to temp file to avoid exposing password in process list +_CH_ENV_FILE=$(mktemp) +chmod 600 "${_CH_ENV_FILE}" +cat > "${_CH_ENV_FILE}" </dev/null 2>&1; then + if (cd "${DBT_DIR}" && dbt run --profiles-dir "${DBT_DIR}"); then + ok "dbt run complete — agg_hourly_zone_trips is now populated." + else + warn "dbt run exited with errors. Run manually:" + warn " cd ${DBT_DIR} && dbt run --profiles-dir \"${DBT_DIR}\"" + fi + else + warn "dbt not available — run manually to populate agg_hourly_zone_trips:" + warn " cd dbt/nyc_taxi_dbt_ch && dbt run" + fi + else + warn "No new trips detected in ClickHouse after 30 seconds." + warn "Troubleshooting tips:" + warn " • Check container logs: docker logs nyc_taxi_ch_producer" + warn " • Verify CLICKHOUSE_HOST is reachable from Docker" + warn " • Check that the producer image supports CLICKHOUSE_HOST env var" + warn " • Try: curl -u default:\${CLICKHOUSE_PASSWORD} https://\${CLICKHOUSE_HOST}:8443/" + fi +else + warn "Could not compare trip counts (before: '${CH_COUNT_BEFORE}', after: '${CH_COUNT_AFTER}')" +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo "" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo -e "${BOLD} Cutover Summary${RESET}" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" +echo -e " ${GREEN}✓${RESET} Snowflake producer stopped: ${BOLD}nyc_taxi_producer${RESET}" +echo -e " ${GREEN}✓${RESET} ClickHouse producer running: ${BOLD}nyc_taxi_ch_producer${RESET}" +echo "" +echo -e " ${BOLD}Live trip data is now writing directly to ClickHouse Cloud.${RESET}" +echo "" +echo " Monitor producer:" +echo " docker logs -f nyc_taxi_ch_producer" +echo "" +info "Verify new trips are arriving in ClickHouse:" +info " docker logs -f nyc_taxi_ch_producer" +info " (new trips should appear every ~10 seconds)" +echo "" +echo -e " ${YELLOW}To roll back (if something goes wrong):${RESET}" +echo " 1. docker stop nyc_taxi_ch_producer" +echo " 2. docker start nyc_taxi_producer (restarts Snowflake producer)" +echo "" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/04_create_dictionary.sql b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/04_create_dictionary.sql new file mode 100644 index 0000000..1aef0a5 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/04_create_dictionary.sql @@ -0,0 +1,26 @@ +-- 04_create_dictionary.sql +-- Creates a ClickHouse dictionary for fast zone lookup via dictGet(). +-- Run after dbt completes (dim_taxi_zones must exist and be populated). +-- Called by setup.sh after Step 5. +-- +-- Usage in SQL: +-- dictGet('analytics.taxi_zones_dict', 'borough', toUInt16(pickup_location_id)) +-- dictGet('analytics.taxi_zones_dict', 'zone', toUInt16(pickup_location_id)) +-- dictGet('analytics.taxi_zones_dict', 'service_zone', toUInt16(pickup_location_id)) + +CREATE OR REPLACE DICTIONARY analytics.taxi_zones_dict +( + location_id UInt16, + zone String, + borough String, + service_zone String +) +PRIMARY KEY location_id +SOURCE(CLICKHOUSE( + TABLE 'dim_taxi_zones' + DB 'analytics' + USER 'default' + PASSWORD '' -- override at runtime with actual password +)) +LAYOUT(HASHED()) +LIFETIME(MIN 300 MAX 600); diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/run_benchmark.sh b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/run_benchmark.sh new file mode 100755 index 0000000..eab22e8 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/scripts/run_benchmark.sh @@ -0,0 +1,342 @@ +#!/usr/bin/env bash +# ============================================================ +# run_benchmark.sh +# Side-by-side benchmark: 7 queries × Snowflake vs ClickHouse +# Runs each query 3 times and reports the median latency. +# +# Standalone usage: +# source .env && source .clickhouse_state +# bash scripts/run_benchmark.sh +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LAB_DIR="$(dirname "${SCRIPT_DIR}")" + +# Source environment files +if [[ -f "${LAB_DIR}/.env" ]]; then + set -a; source "${LAB_DIR}/.env"; set +a +fi +if [[ -f "${LAB_DIR}/.clickhouse_state" ]]; then + set -a; source "${LAB_DIR}/.clickhouse_state"; set +a +fi + +# --------------------------------------------------------------------------- +# Colors & log helpers +# --------------------------------------------------------------------------- +BOLD="\033[1m"; RESET="\033[0m" +GREEN="\033[1;32m"; RED="\033[1;31m"; YELLOW="\033[1;33m"; BLUE="\033[1;34m" + +info() { echo -e "${BLUE} ${*}${RESET}"; } +ok() { echo -e "${GREEN} ✓ ${*}${RESET}"; } +warn() { echo -e "${YELLOW} ⚠ ${*}${RESET}"; } +die() { echo -e "${RED}${BOLD} ✗ ${*}${RESET}" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Validate required vars +# --------------------------------------------------------------------------- +: "${CLICKHOUSE_HOST:?CLICKHOUSE_HOST is not set. Source .clickhouse_state first.}" +: "${CLICKHOUSE_PASSWORD:?CLICKHOUSE_PASSWORD is not set.}" +CLICKHOUSE_PORT="${CLICKHOUSE_PORT:-8443}" +CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" + +: "${SNOWFLAKE_ORG:?SNOWFLAKE_ORG is not set. Source .env first.}" +: "${SNOWFLAKE_ACCOUNT:?SNOWFLAKE_ACCOUNT is not set.}" +: "${SNOWFLAKE_USER:?SNOWFLAKE_USER is not set.}" +: "${SNOWFLAKE_PASSWORD:?SNOWFLAKE_PASSWORD is not set.}" + +# --------------------------------------------------------------------------- +# Locate snowsql +# --------------------------------------------------------------------------- +SNOWSQL_CMD="" +if command -v snowsql >/dev/null 2>&1; then SNOWSQL_CMD="snowsql" +elif [[ -x "/Applications/SnowSQL.app/Contents/MacOS/snowsql" ]]; then SNOWSQL_CMD="/Applications/SnowSQL.app/Contents/MacOS/snowsql" +fi +[[ -n "${SNOWSQL_CMD}" ]] || die "snowsql not found. Install from https://docs.snowflake.com/en/user-guide/snowsql-install-config" + +# --------------------------------------------------------------------------- +# Portable millisecond timestamp (macOS-safe) +# --------------------------------------------------------------------------- +ms_now() { + python3 -c "import time; print(int(time.time() * 1000))" +} + +# --------------------------------------------------------------------------- +# Timing helpers +# --------------------------------------------------------------------------- +time_query_snowflake() { + local query="$1" + local start end + start=$(ms_now) + SNOWSQL_PWD="${SNOWFLAKE_PASSWORD}" "${SNOWSQL_CMD}" \ + --accountname "${SNOWFLAKE_ORG}-${SNOWFLAKE_ACCOUNT}" \ + --username "${SNOWFLAKE_USER}" \ + --rolename ANALYST_ROLE \ + -q "USE WAREHOUSE ANALYTICS_WH; ${query}" \ + --option output_format=plain --option friendly=false >/dev/null 2>&1 + end=$(ms_now) + echo $(( end - start )) +} + +time_query_clickhouse() { + local query="$1" + local start end + start=$(ms_now) + curl -s --fail \ + "https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/" \ + -u "${CLICKHOUSE_USER}:${CLICKHOUSE_PASSWORD}" \ + --data-urlencode "query=${query}" \ + --data-urlencode "database=nyc_taxi_ch" >/dev/null 2>&1 + end=$(ms_now) + echo $(( end - start )) +} + +# --------------------------------------------------------------------------- +# Median of 3 values +# --------------------------------------------------------------------------- +median_of_3() { + local a=$1 b=$2 c=$3 + if (( a <= b && b <= c )); then echo "$b" + elif (( a <= c && c <= b )); then echo "$c" + elif (( b <= a && a <= c )); then echo "$a" + elif (( b <= c && c <= a )); then echo "$c" + elif (( c <= a && a <= b )); then echo "$a" + else echo "$b" + fi +} + +# --------------------------------------------------------------------------- +# Format milliseconds as human-readable seconds string (e.g. "4.2s") +# --------------------------------------------------------------------------- +fmt_ms() { + local ms=$1 + local sec=$(( ms / 1000 )) + local frac=$(( (ms % 1000) / 100 )) + echo "${sec}.${frac}s" +} + +# --------------------------------------------------------------------------- +# Run one query on both systems, 3 times each, return median +# Sets globals: LAST_SF_MS LAST_CH_MS LAST_SF_SKIPPED +# --------------------------------------------------------------------------- +LAST_SF_MS=0 +LAST_CH_MS=0 +LAST_SF_SKIPPED=false + +benchmark_query() { + local label="$1" + local sf_query="$2" + local ch_query="$3" + local allow_sf_fail="${4:-false}" + + LAST_SF_SKIPPED=false + + info " Running ${label} on Snowflake (3 runs)…" + if [[ "${allow_sf_fail}" == "true" ]]; then + SF_R1=0; SF_R2=0; SF_R3=0 + if ! SF_R1=$(time_query_snowflake "${sf_query}" 2>/dev/null); then + LAST_SF_MS=0; LAST_SF_SKIPPED=true + warn " ${label} Snowflake query failed — marking as N/A" + else + SF_R2=$(time_query_snowflake "${sf_query}" 2>/dev/null) || SF_R2=${SF_R1} + SF_R3=$(time_query_snowflake "${sf_query}" 2>/dev/null) || SF_R3=${SF_R2} + LAST_SF_MS=$(median_of_3 "${SF_R1}" "${SF_R2}" "${SF_R3}") + fi + else + SF_R1=$(time_query_snowflake "${sf_query}") + SF_R2=$(time_query_snowflake "${sf_query}") + SF_R3=$(time_query_snowflake "${sf_query}") + LAST_SF_MS=$(median_of_3 "${SF_R1}" "${SF_R2}" "${SF_R3}") + fi + + info " Running ${label} on ClickHouse (3 runs)…" + CH_R1=$(time_query_clickhouse "${ch_query}") + CH_R2=$(time_query_clickhouse "${ch_query}") + CH_R3=$(time_query_clickhouse "${ch_query}") + LAST_CH_MS=$(median_of_3 "${CH_R1}" "${CH_R2}" "${CH_R3}") +} + +# --------------------------------------------------------------------------- +# Query definitions +# --------------------------------------------------------------------------- + +SF_Q1="SELECT DATE_TRUNC('hour', pickup_at) AS hour_bucket, pickup_borough, COUNT(*) AS trips, SUM(total_amount_usd) AS revenue FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS WHERE pickup_at >= DATEADD('day', -7, CURRENT_TIMESTAMP()) AND pickup_borough IS NOT NULL GROUP BY 1, 2 ORDER BY 1 DESC, revenue DESC LIMIT 100" +CH_Q1="SELECT toStartOfHour(pickup_at) AS hour_bucket, pickup_borough, count() AS trips, sum(total_amount_usd) AS revenue FROM analytics.fact_trips FINAL WHERE pickup_at >= now() - INTERVAL 7 DAY AND pickup_borough != '' GROUP BY hour_bucket, pickup_borough ORDER BY hour_bucket DESC, revenue DESC LIMIT 100" + +SF_Q2="SELECT pickup_at::DATE AS trip_date, COUNT(*) AS trips, AVG(AVG(trip_distance_miles)) OVER (ORDER BY pickup_at::DATE ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS rolling_7d_avg FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS GROUP BY 1 ORDER BY 1 DESC LIMIT 365" +CH_Q2="SELECT toDate(pickup_at) AS trip_date, count() AS trips, avg(avg(trip_distance_miles)) OVER (ORDER BY trip_date ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS rolling_7d_avg FROM analytics.fact_trips FINAL GROUP BY trip_date ORDER BY trip_date DESC LIMIT 365" + +SF_Q3="SELECT trip_id, pickup_borough, total_amount_usd, ROW_NUMBER() OVER (PARTITION BY pickup_borough ORDER BY total_amount_usd DESC) AS rn FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS WHERE pickup_at::DATE = CURRENT_DATE() - 1 QUALIFY rn <= 10" +CH_Q3="SELECT trip_id, pickup_borough, total_amount_usd, rn FROM (SELECT trip_id, pickup_borough, total_amount_usd, row_number() OVER (PARTITION BY pickup_borough ORDER BY total_amount_usd DESC) AS rn FROM analytics.fact_trips FINAL WHERE toDate(pickup_at) = today() - 1) WHERE rn <= 10" + +SF_Q4="SELECT ROUND(TRIP_METADATA:driver.rating::FLOAT, 1) AS rating_bucket, COUNT(*) AS trips, AVG(TOTAL_AMOUNT) AS avg_fare FROM NYC_TAXI_DB.RAW.TRIPS_RAW WHERE TRIP_METADATA:driver.rating IS NOT NULL GROUP BY 1 ORDER BY 1" +CH_Q4="SELECT round(JSONExtractFloat(trip_metadata, 'driver', 'rating'), 1) AS rating_bucket, count() AS trips, avg(total_amount) AS avg_fare FROM nyc_taxi_ch.trips_raw WHERE JSONExtractFloat(trip_metadata, 'driver', 'rating') > 0 GROUP BY rating_bucket ORDER BY rating_bucket" + +SF_Q5="SELECT CASE WHEN TRIP_METADATA:app.surge_multiplier::FLOAT >= 2.0 THEN 'High (2x+)' WHEN TRIP_METADATA:app.surge_multiplier::FLOAT >= 1.5 THEN 'Medium (1.5-2x)' WHEN TRIP_METADATA:app.surge_multiplier::FLOAT > 1.0 THEN 'Low (1-1.5x)' ELSE 'No Surge' END AS surge_cat, COUNT(*) AS trips, ROUND(AVG(TOTAL_AMOUNT), 2) AS avg_fare FROM NYC_TAXI_DB.RAW.TRIPS_RAW GROUP BY 1 ORDER BY 2 DESC" +CH_Q5="SELECT CASE WHEN JSONExtractFloat(trip_metadata,'app','surge_multiplier') >= 2.0 THEN 'High (2x+)' WHEN JSONExtractFloat(trip_metadata,'app','surge_multiplier') >= 1.5 THEN 'Medium (1.5-2x)' WHEN JSONExtractFloat(trip_metadata,'app','surge_multiplier') > 1.0 THEN 'Low (1-1.5x)' ELSE 'No Surge' END AS surge_cat, count() AS trips, round(avg(total_amount),2) AS avg_fare FROM nyc_taxi_ch.trips_raw GROUP BY surge_cat ORDER BY trips DESC" + +SF_Q6="SELECT DATE_TRUNC('hour', pickup_at) AS hour_bucket, pickup_location_id AS zone_id, COUNT(*) AS trips, SUM(total_amount_usd) AS revenue FROM NYC_TAXI_DB.ANALYTICS.FACT_TRIPS WHERE pickup_at >= DATEADD('hour', -24, CURRENT_TIMESTAMP()) GROUP BY 1, 2 ORDER BY 1 DESC, revenue DESC LIMIT 100" +CH_Q6="SELECT toStartOfHour(pickup_at) AS hour_bucket, pickup_location_id AS zone_id, count() AS trips, sum(total_amount_usd) AS revenue FROM analytics.fact_trips FINAL WHERE pickup_at >= now() - INTERVAL 24 HOUR GROUP BY hour_bucket, zone_id ORDER BY hour_bucket DESC, revenue DESC LIMIT 100" + +SF_Q7="SELECT METADATA\$ACTION, COUNT(*) AS changes FROM NYC_TAXI_DB.RAW.TRIPS_CDC_STREAM GROUP BY 1" +CH_Q7="SELECT count() AS total_trips, max(ingested_at) AS latest_ingest, countIf(ingested_at >= now() - INTERVAL 5 MINUTE) AS recent_trips FROM nyc_taxi_ch.trips_raw" + +# --------------------------------------------------------------------------- +# Query labels / descriptions +# --------------------------------------------------------------------------- +declare -a Q_LABELS=( + "Q1 Hourly revenue by borough" + "Q2 Rolling 7-day avg distance" + "Q3 Top 10 trips (QUALIFY→subquery)" + "Q4 Driver ratings (JSON flatten)" + "Q5 Surge pricing (VARIANT)" + "Q6 Hourly aggregation (MERGE→RMT)" + "Q7 CDC/live data freshness" +) +declare -a Q_IDS=( Q1 Q2 Q3 Q4 Q5 Q6 Q7 ) +declare -a Q_DESC=( + "Hourly revenue by borough" + "Rolling 7-day avg distance" + "Top 10 trips (QUALIFY→subquery)" + "Driver ratings (JSON flatten)" + "Surge pricing (VARIANT)" + "Hourly aggregation (MERGE→RMT)" + "CDC/live data freshness" +) + +DIVIDER="━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +SEP="────────────────────────────────────────────────────────────────────────" + +echo "" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo -e "${BOLD} NYC Taxi Lab — Query Benchmark: Snowflake vs ClickHouse${RESET}" +echo -e "${BOLD} (median of 3 runs each)${RESET}" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" +info "Note: Snowflake times include CLI session setup (~1-3s per query). For query-engine-only comparison, use the Snowflake UI Query History." +echo "" + +# --------------------------------------------------------------------------- +# Run all 7 benchmarks +# --------------------------------------------------------------------------- + +declare -a SF_RESULTS=() +declare -a CH_RESULTS=() +declare -a SF_SKIPPED=() + +# Q1 +benchmark_query "Q1" "${SF_Q1}" "${CH_Q1}" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# Q2 +benchmark_query "Q2" "${SF_Q2}" "${CH_Q2}" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# Q3 +benchmark_query "Q3" "${SF_Q3}" "${CH_Q3}" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# Q4 +benchmark_query "Q4" "${SF_Q4}" "${CH_Q4}" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# Q5 +benchmark_query "Q5" "${SF_Q5}" "${CH_Q5}" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# Q6 +benchmark_query "Q6" "${SF_Q6}" "${CH_Q6}" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# Q7 — Snowflake CDC stream may fail if stream is consumed; allow failure +benchmark_query "Q7" "${SF_Q7}" "${CH_Q7}" "true" +SF_RESULTS+=( "${LAST_SF_MS}" ); CH_RESULTS+=( "${LAST_CH_MS}" ); SF_SKIPPED+=( "${LAST_SF_SKIPPED}" ) + +# --------------------------------------------------------------------------- +# Build result table & CSV +# --------------------------------------------------------------------------- +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +CSV_FILE="${SCRIPT_DIR}/benchmark_results_${TIMESTAMP}.csv" + +{ + echo "query,description,snowflake_ms,clickhouse_ms,speedup" +} > "${CSV_FILE}" + +echo "" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo -e "${BOLD} NYC Taxi Lab — Query Benchmark: Snowflake vs ClickHouse${RESET}" +echo -e "${BOLD} (median of 3 runs each)${RESET}" +echo -e "${BOLD}${DIVIDER}${RESET}" +printf "%-38s %-12s %-12s %s\n" "Query" "Snowflake" "ClickHouse" "Speedup" +echo "${SEP}" + +TOTAL_SF=0 +TOTAL_CH=0 +TOTAL_SF_VALID=0 +SF_SKIPPED_ANY=false + +for i in 0 1 2 3 4 5 6; do + sf_ms="${SF_RESULTS[$i]}" + ch_ms="${CH_RESULTS[$i]}" + skipped="${SF_SKIPPED[$i]}" + label="${Q_LABELS[$i]}" + qid="${Q_IDS[$i]}" + desc="${Q_DESC[$i]}" + + if [[ "${ch_ms}" -gt 0 ]]; then + ch_fmt=$(fmt_ms "${ch_ms}") + else + ch_fmt="ERR" + fi + + if [[ "${skipped}" == "true" ]] || [[ "${sf_ms}" -eq 0 ]]; then + sf_fmt="N/A" + speedup_str="N/A" + SF_SKIPPED_ANY=true + echo "query,${desc},N/A,${ch_ms},N/A" >> "${CSV_FILE}" + printf "%-38s %-12s %-12s %s\n" "${label}" "${sf_fmt}" "${ch_fmt}" "${speedup_str}" + else + sf_fmt=$(fmt_ms "${sf_ms}") + if [[ "${ch_ms}" -gt 0 ]]; then + speedup=$(( sf_ms / ch_ms )) + speedup_str="${speedup}x" + else + speedup=0 + speedup_str="N/A" + fi + echo "${qid},${desc},${sf_ms},${ch_ms},${speedup}" >> "${CSV_FILE}" + printf "%-38s %-12s %-12s %s\n" "${label}" "${sf_fmt}" "${ch_fmt}" "${speedup_str}" + TOTAL_SF=$(( TOTAL_SF + sf_ms )) + TOTAL_CH=$(( TOTAL_CH + ch_ms )) + (( TOTAL_SF_VALID++ )) || true + fi +done + +echo "${SEP}" + +# Totals row +TOTAL_SF_FMT=$(fmt_ms "${TOTAL_SF}") +TOTAL_CH_FMT=$(fmt_ms "${TOTAL_CH}") +SPEEDUP_NOTE="" +if [[ "${SF_SKIPPED_ANY}" == "true" ]]; then + # Only sum queries where both systems were measured + SPEEDUP_NOTE=" (excluding N/A queries)" +fi +if [[ "${TOTAL_CH}" -gt 0 ]]; then + TOTAL_SPEEDUP=$(( TOTAL_SF / TOTAL_CH )) + TOTAL_SPEEDUP_STR="${TOTAL_SPEEDUP}x avg${SPEEDUP_NOTE}" +else + TOTAL_SPEEDUP_STR="N/A" +fi + +printf "%-38s %-12s %-12s %s\n" "Total" "${TOTAL_SF_FMT}" "${TOTAL_CH_FMT}" "${TOTAL_SPEEDUP_STR}" +echo -e "${BOLD}${DIVIDER}${RESET}" +echo "" +echo -e " Results written to: ${BOLD}${CSV_FILE}${RESET}" +echo "" + +# Also append totals to CSV +echo "TOTAL,All queries,${TOTAL_SF},${TOTAL_CH},${TOTAL_SPEEDUP:-N/A}" >> "${CSV_FILE}" + +ok "Benchmark complete. CSV saved to: ${CSV_FILE}" +echo "" diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh new file mode 100755 index 0000000..6d59cb6 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# ============================================================ +# setup.sh — Provision ClickHouse Cloud service for Part 3 +# +# This script does ONE thing: provision the ClickHouse Cloud cluster +# via Terraform and write the connection details to .clickhouse_state. +# +# Everything else (dbt, migration script, verification, Superset, benchmark) +# is done manually following the steps in the playbook at +# https://workshop.demohouse.cloud/docs/snowflake-migration — because those +# are the migration decisions you should make deliberately, not skip. +# +# Prerequisites: +# - terraform >= 1.6 +# - Completed Part 2 (02-plan-and-design) with migration-plan.md +# +# Usage: +# cp .env.example .env && vim .env # fill in your CH Cloud credentials +# source .env && ./setup.sh +# ============================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Auto-source .env if present +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + set -a + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.env" + set +a +fi + +# Auto-source .clickhouse_state if present (written after Terraform apply) +if [[ -f "${SCRIPT_DIR}/.clickhouse_state" ]]; then + set -a + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.clickhouse_state" + set +a +fi + +TERRAFORM_DIR="${SCRIPT_DIR}/terraform" + +# ── Helpers ─────────────────────────────────────────────────── +BOLD="\033[1m"; RESET="\033[0m" +BLUE="\033[1;34m"; GREEN="\033[1;32m"; YELLOW="\033[1;33m"; RED="\033[1;31m" + +log() { echo -e "\n${BLUE}${BOLD}▶ $*${RESET}"; _STEP_START=$(date +%s); _CURRENT_STEP="$*"; } +ok() { local s=$(( $(date +%s) - ${_STEP_START:-$(date +%s)} )); echo -e "${GREEN}${BOLD}✓ $*${RESET} (${s}s)"; } +info() { echo -e " ${BOLD}·${RESET} $*"; } +warn() { echo -e "${YELLOW} ⚠ $*${RESET}"; } +die() { echo -e "\n${RED}${BOLD}✗ ERROR: $*${RESET}\n"; exit 1; } + +_CURRENT_STEP="initializing" +_STEP_START=$(date +%s) + +on_error() { + echo -e "\n${RED}${BOLD}Setup FAILED at: ${_CURRENT_STEP}${RESET}" + echo " Troubleshooting:" + echo " • Verify CLICKHOUSE_ORG_ID, CLICKHOUSE_TOKEN_KEY, CLICKHOUSE_TOKEN_SECRET are set" + echo " • Check API key permissions at https://console.clickhouse.cloud → Settings → API Keys" + echo " • Run manually: cd terraform && terraform plan" +} +trap 'on_error' ERR + +# ── Header ───────────────────────────────────────────────────── +echo -e "\n${BLUE}${BOLD}NYC Taxi Migration Lab — Part 3: ClickHouse Cloud${RESET}" +echo "────────────────────────────────────────────────────" + +# ── Part 2 prerequisite check (soft gate) ───────────────────── +PART2_PLAN="${SCRIPT_DIR}/../02-plan-and-design/migration-plan.md" +if [[ ! -f "${PART2_PLAN}" ]]; then + warn "Part 2 migration-plan.md not found at ${PART2_PLAN}" + warn "Strongly recommended: complete Part 2 (Plan & Design) before continuing." + warn "Proceeding anyway — but you may encounter design decisions without context." +else + COMPLETED=$(grep -c '\- \[x\]' "${PART2_PLAN}" 2>/dev/null || echo 0) + TOTAL=$(grep -c '\- \[[ x]\]' "${PART2_PLAN}" 2>/dev/null || echo 4) + if [[ "${COMPLETED}" -ge "${TOTAL}" ]]; then + info "Part 2 migration plan: ${COMPLETED}/${TOTAL} sections complete ✓" + else + warn "Part 2 migration plan: ${COMPLETED}/${TOTAL} sections complete — consider finishing worksheets first." + fi +fi + +# ── Prerequisites check ─────────────────────────────────────── +info "Checking prerequisites..." +command -v terraform >/dev/null 2>&1 || die "terraform not found → https://developer.hashicorp.com/terraform/downloads" + +MISSING_VARS=() +[[ -z "${CLICKHOUSE_ORG_ID:-}" ]] && MISSING_VARS+=("CLICKHOUSE_ORG_ID") +[[ -z "${CLICKHOUSE_TOKEN_KEY:-}" ]] && MISSING_VARS+=("CLICKHOUSE_TOKEN_KEY") +[[ -z "${CLICKHOUSE_TOKEN_SECRET:-}" ]] && MISSING_VARS+=("CLICKHOUSE_TOKEN_SECRET") +[[ -z "${CLICKHOUSE_PASSWORD:-}" ]] && MISSING_VARS+=("CLICKHOUSE_PASSWORD") + +if [[ ${#MISSING_VARS[@]} -gt 0 ]]; then + echo -e "${RED}${BOLD}Missing required environment variables:${RESET}" + for v in "${MISSING_VARS[@]}"; do echo " $v"; done + echo "" + echo " Copy .env.example → .env and fill in your ClickHouse Cloud credentials." + exit 1 +fi + +info "Prerequisites OK." + +# ── Terraform: Provision ClickHouse Cloud service ───────────── +log "Provisioning ClickHouse Cloud service via Terraform" + +info "Writing terraform.tfvars..." +cat > "${TERRAFORM_DIR}/terraform.tfvars" < "${SCRIPT_DIR}/.clickhouse_state" <>> Waiting for Superset to be ready..." +until curl -sf "${SUPERSET_URL}/health" > /dev/null; do + sleep 3 +done +echo ">>> Superset is up." + +# ── Authenticate ────────────────────────────────────────────────────────────── +LOGIN_RESPONSE=$(curl -s -c /tmp/superset_cookies.txt \ + -X POST "${SUPERSET_URL}/api/v1/security/login" \ + -H "Content-Type: application/json" \ + -d "{\"username\": \"${ADMIN_USER}\", \"password\": \"${ADMIN_PASS}\", \"provider\": \"db\", \"refresh\": true}") + +ACCESS_TOKEN=$(echo "${LOGIN_RESPONSE}" | python3 -c "import sys, json; print(json.load(sys.stdin)['access_token'])" 2>/dev/null || true) +if [ -z "${ACCESS_TOKEN}" ]; then + echo "ERROR: Superset login failed. Check SUPERSET_ADMIN_USER / SUPERSET_ADMIN_PASSWORD." + echo " Response: ${LOGIN_RESPONSE}" + exit 1 +fi +echo ">>> Authenticated." + +AUTH_HEADER="Authorization: Bearer ${ACCESS_TOKEN}" + +CSRF_TOKEN=$(curl -s \ + -b /tmp/superset_cookies.txt \ + -c /tmp/superset_cookies.txt \ + -H "${AUTH_HEADER}" \ + "${SUPERSET_URL}/api/v1/security/csrf_token/" \ + | python3 -c "import sys, json; print(json.load(sys.stdin)['result'])") +CSRF_HEADER="X-CSRFToken: ${CSRF_TOKEN}" +echo ">>> CSRF token obtained." + +# ── Patch ZIP: replace database URI with current credentials ────────────────── +# The exported database YAML has the original host and XXXXXXXXXX as the +# password placeholder. Rewrite sqlalchemy_uri before importing. +echo ">>> Patching dashboard export with current ClickHouse credentials..." +PATCHED_ZIP=$(mktemp /tmp/superset_import_XXXXXX.zip) + +python3 - "${EXPORT_ZIP}" "${CLICKHOUSE_URI}" "${PATCHED_ZIP}" <<'PYEOF' +import sys, zipfile, re, yaml, json + +src_zip, new_uri, dst_zip = sys.argv[1], sys.argv[2], sys.argv[3] + +def clean_position(pos): + """ + The exported ZIP has two sets of chart entries per dashboard: + 1. Ghost rows (ROW-1, ROW-2, …): proper 2-per-row layout, width=6 each, + but chart entries have no uuid so they break on import. + 2. Duplicate row (ROW-N-XXXXXX): all charts crammed into one row at + width=4, but these entries DO have uuid + sliceName. + + Fix: patch the ghost entries with the uuid/sliceName from the duplicate row, + then drop the duplicate row so only the properly-laid-out rows remain. + """ + # 1. Build chartId → {uuid, sliceName} from UUID-bearing entries + uuid_map = {} + for v in pos.values(): + if isinstance(v, dict) and v.get('type') == 'CHART': + meta = v.get('meta', {}) + if meta.get('uuid') and meta.get('chartId') is not None: + uuid_map[meta['chartId']] = { + 'uuid': meta['uuid'], + 'sliceName': meta.get('sliceName', ''), + } + + # 2. Identify ghost chart keys (CHART-, no uuid) + ghost_keys = { + k for k, v in pos.items() + if isinstance(v, dict) + and v.get('type') == 'CHART' + and re.fullmatch(r'CHART-\d+', k) + and not v.get('meta', {}).get('uuid') + } + + # 3. Patch ghost entries with uuid + sliceName; keep their layout (width=6) + for k in ghost_keys: + chart_id = pos[k]['meta']['chartId'] + if chart_id in uuid_map: + pos[k]['meta']['uuid'] = uuid_map[chart_id]['uuid'] + pos[k]['meta']['sliceName'] = uuid_map[chart_id]['sliceName'] + + # 4. Identify UUID-bearing chart keys (the duplicates) + uuid_chart_keys = { + k for k, v in pos.items() + if isinstance(v, dict) + and v.get('type') == 'CHART' + and not re.fullmatch(r'CHART-\d+', k) + and v.get('meta', {}).get('uuid') + } + + # 5. Remove uuid-bearing chart keys from every parent's children list + for v in pos.values(): + if isinstance(v, dict) and 'children' in v: + v['children'] = [c for c in v['children'] if c not in uuid_chart_keys] + + # 6. Remove uuid-bearing chart entries from the position dict + for k in uuid_chart_keys: + del pos[k] + + # 7. Drop any ROW that is now empty (the duplicate all-in-one row) + empty_rows = { + k for k, v in pos.items() + if isinstance(v, dict) and v.get('type') == 'ROW' and not v.get('children') + } + for v in pos.values(): + if isinstance(v, dict) and 'children' in v: + v['children'] = [c for c in v['children'] if c not in empty_rows] + for k in empty_rows: + del pos[k] + + return pos + +with zipfile.ZipFile(src_zip, 'r') as zin, \ + zipfile.ZipFile(dst_zip, 'w', zipfile.ZIP_DEFLATED) as zout: + for item in zin.infolist(): + data = zin.read(item.filename) + if 'databases/' in item.filename and item.filename.endswith('.yaml'): + text = data.decode('utf-8') + text = re.sub( + r'^sqlalchemy_uri:.*$', + f'sqlalchemy_uri: {new_uri}', + text, + flags=re.MULTILINE + ) + data = text.encode('utf-8') + elif 'dashboards/' in item.filename and item.filename.endswith('.yaml'): + doc = yaml.safe_load(data) + if 'position' in doc and isinstance(doc['position'], dict): + doc['position'] = clean_position(doc['position']) + data = yaml.dump(doc, allow_unicode=True, sort_keys=False).encode('utf-8') + zout.writestr(item, data) +PYEOF + +echo " Done." + +# ── Import ──────────────────────────────────────────────────────────────────── +echo ">>> Importing dashboards (overwrite=true)..." +response=$(curl -s -w "\n%{http_code}" \ + -X POST "${SUPERSET_URL}/api/v1/dashboard/import/" \ + -b /tmp/superset_cookies.txt \ + -H "${AUTH_HEADER}" \ + -H "${CSRF_HEADER}" \ + -F "formData=@${PATCHED_ZIP};type=application/zip" \ + -F "overwrite=true") +http_code=$(echo "${response}" | tail -1) +body=$(echo "${response}" | sed '$d') + +rm -f "${PATCHED_ZIP}" + +if [ "${http_code}" = "200" ]; then + echo "" + echo "============================================================" + echo " ClickHouse Superset Setup Complete" + echo " Superset URL: ${SUPERSET_URL}" + echo "" + echo " Dashboards imported:" + echo " 1. CH — Operations Command Center" + echo " 2. CH — Executive Weekly Report" + echo " 3. CH — Driver Quality Analytics" + echo " 4. CH — Capabilities Showcase" + echo "" + echo " Login: ${SUPERSET_URL} / ${ADMIN_USER} / ${ADMIN_PASS}" + echo "============================================================" +else + echo "ERROR: Import failed (HTTP ${http_code}):" + echo "${body}" + exit 1 +fi diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/superset/dashboards/dashboard_export_20260401T084808.zip b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/superset/dashboards/dashboard_export_20260401T084808.zip new file mode 100644 index 0000000000000000000000000000000000000000..d74c76df8a044861bbedb4aae968d37f55470eae GIT binary patch literal 23747 zcmb?@1yt2p*EgL~f^>IFNq09$_ocfV>FzF(mhSFGy1Pq4KuSPB`n%}7<1?=3`DR>e z?pjAKOYS-Q{(rmA-dkP@90CsH=`}1_p!NODAO8>me;VpJnCjc;*&70kTy1Ua9RW-X zOsotn42(()>@4gI>@Vg2=f`RO$;S=#9QE||9E=>kZ-fT?2+Vgn1j0#(i9kU>R)9<5 z_coGM7Xm2hxtallEX@oo#BH1$jDT-#oDAvR^sFqkW8~yMFg)u$xksBkUd^OeLWcFN zg{P3k<@X_Gvr`X9jUdG@L0(U{oC;#i%<%F$tFa;yf`+XN22Fg+-Co**dYIiV|6m50<4{`Wk;gw)D+49ol{Hx$Bth zShyzQXze2jsV9tZ$u07@0brfRSXFr0Q8f}m-M_7>uGP$Pm2SysRU0r}w=Ky*Vz<&>N|Mq?RMx33|d z8cU87=o6FYKw~^yPFgVNbuSybl_yd2`ZABfzsn^qqh+^=Yq6REs#pz-$KFfiMm>KT zGKKm&0X)St1xsOYvKbrkAhI}_p{THuHJdxqUAQujqSJQ8#Hjd<@Qz(XM#xT8UY2l@ zq!GLewg}WjpZy)RO+rwdE^l}P2@+^#+~Xmg2zDPNA#%dEFtAh zYlVj0QWL71Qol^Osn}K|BXIu8PJW?r6u^QzL%!Nzblrc)JOf)Rdm+s+Gkm%i#L&B$ zm$;H9D%@rC?bxGmx$fnpJmuelqhJE__`2eGAJRU)hs&=PMI~aH6W-=ENaqrpBp%7w7%fy z`Fb$wJdgD5XyOE7wDSa}E5~_Z&pGRe%!ku%`rM=XfmZaL0f}0I9wMi-498y1{!y;I zR&StwgSN3|i=MAzgu2YvxwU%4u?AcU5RtS@2-qYhYs&#ucVs#QoVS;5f1NjnA-IO2389h5P5kb%MYiw0b24c|gO ziTY1laD&Stj|dbXWS{`){7V9)XJP^{wQ;hybOXGxu{HuYKFQAysY&Fq0ZL7f%RQWx z1UzLWnR;j&q!}E9X}m~AD5_g^s6y@#64_A2OQT7@&-DW_&!C!PUq-ultmO#9C-M_# zcgZ|!!CRy$5RubK6xM%bBQpb%9H738T--yB4<^2|hg6FqB;*;fBWbO_AMiyIK~~K# zt}aXp$5445>fR9Lb(VW(3R~fd-K=X7C;;HMM^`}3#3!O=As}SX-J?GZS1InLXhBbF zt}joBM~x(GiK=qZ4ANMqh)oLc2`^j#JO?(N6XOp>Ni<6@%-s?lxBRe4iuZ; zuxhvM#Y&U?U0HfFJ#m8+$80`O2@LefFdRuNvq;e# zW$SDD_MnIQT`CN|TZn%?XjfgYNEv*8na(0XWyDe17!!YQe8+t<%8sZXs3jC+;Oy3m z9`k~FmgW+?W@VUgR*N&)F1CbmI!_@(m_}Nok!7BX)K`HcN(0@kSQ~s-LO+((mx7<0 z*u^Z!ZBdj?0IN22y~nHe?V9w z9|3|QH`x&?T@@w!s|k2BZ?<3P64uDL<4(!@2t7c_DC|twhQo=>-kHa2zYU|Gl$r`DH!Rz2TQl@F#J zfl<^uAZGxH1jZvIXrtdg#6MgB8IhhinwBls^YFSDhh^o8Tc*`Y`PY3vpO$R zsh7z+Xgabbk8m35LF#z54WUc61krw5DPKVz0 zRaHgm11H#IPVB zPf@|&1_FQjSO42D0Ar_6o2M12FR!%?s1UA{%6BVEKu<=Shkjg7TlIYXhb6WN2JL@j=WPVH-9 zR~``!6ojmrAcldsN(gJj2{yP4_*UpfxymAbcF%~aj`@632%S2rTWXWP5H89SK-}i| z3S$M&{)I56$wfISrQgjCn-QPpyM|AWbtt8jGlLis8@$G_E}JZ4^(|luN?Pb8Xjha> zwDWf4t>}jhWpGwDEPbpaH)czviDC@Sd{wIFvnv{vC4dU1WY8p@9Mh$dgX9=19@@pk zrZ)_ukT?k(Csq#TW1qrtSk*@s9Ici==_iVP5?jKXv#A-3!wl3TtY2|`v)vP}+RD1Q zRDUcfS+~JN$3v|{EuOf;hwQ5&f}#{#ak{Oois>*2I%{5YW~MNQWaMgBeko=AdN>%3 zL-^V(L*gXRdo;Pqj}Zn2iQ0Cj38{Qjq`G?O!h^ACbj#Y(!V_>i?`B}{v)CyMA4 z7@ximkFCJL*gHt}KM6uiUd`azivmFagvW3@?5ItvHA=;2uYd%`BFXKw_MH?lW)k~K>s2Y{WE-uHJ(!e%DG z$m@sjDapuvH#Mqi?&hkg)K=59`;m~%;XMjU8g}7 zCroXbzC{mw(eybMP11~tT4!5u5q)fWCTkf2AN}dGZ%71!%{|OX5V6BrZp8tz%(Fnt&UKtdHW)xV`;3p z4;sbHPBk?Yb3XOG)Kck%F3*1SjxtVdz+`c!Xkp6JYS${ zap6$c+CjwkaOhsEGCX_T`x)eZ^FXlZkox$w7@3+4C8 z?~RZg;j4v(;3Ksd`fWnLSvt9RQqNYf#3 zhwUGHc8400LO@P+zQulhEC$M_#~3uNi|dNTN%JBBQ=V|Ad8?i=%Ibq}Ao}$dt?SoyuoRq)G(~CChp6TUq2Y2o2b}?71$wg_O-V zHc3{=tEKH`9WrDiG!5pR2!RXw?g9ZU)#yynTW23+^KuovC z@41Dae*GbJRqhRt>o4&^KotKat~b^*aQshaK1D_jhY>mG)={)zzg3dBd*6RTlM|LXB4PQu#vX`7O z??RC|`y$^ZsGKdEVg(3h?JALo=TPKDNyg>{7J>=UmBw;@HJ+}=r*crU-C#Bpa5mj+ zo{2B=mw?5dAr?P|_b#|$f=A?BHvw~25#YZ}txKfNGQfXv#$xY>9M9AkA`a-;X}PyR0$Dc zbaC@XCnqM;Ec4VGOe8lR{HW+KLb~Fz=5~3lPAZj_hS^? z&61v8gZUQtN7s%7uae_TBBJF=8Am;XjY*Cfk19q>n%7BJ8?`fRv@uO=MgcOyEMIB4?>) zom}^&n5OpTXDSCMd|}JCik#En&WY9zNSQQ)F_cSKLjMHINfp2wW5Ll4Z$eA=nRZdr zb|6dYAV%cn168Ma&bJn@%nqG&*?X-TJB5zQOHQ z4C3AJNbN9x2m26T{`yP$j#gO9XlwfYg=iKjV}$T{rq`vyAnS~jDXQi#eZ?JzgFqFEwRaHPydk!=(|Drdi@f11>n*l{m+Q!Dh$@aTJSyj@u zSz79Vf{J4aCHGuP3n-=*>t~R-3bQe-nDQp+1yLQ z!!px-{IsnexxkNlpeH0vCkCJa%ak+{M^omV+?4FAl%SdVI)57Fwb8pbT9_&WE0Sf{ zq`pjbN|YHEMV1H$q>Nw`zWqLB5Oqwb%`0Wt0Za=r4}`Z}9`>&|+SDlp*+1FRTOJ^z z<>Jt!?5nxPd__TJnJL4^kNu#A(X3VeW&ZQBlu^rTn&fBa`9;PX&vWA%mf@pgAURUv z-#kt(lQd#^*|pO7EdwT*wW*iQ9B2zK?;RW(MyxdAm9Xu96-&?TyI0Z z>O|kd?SzAfi{pJ1Odri(7x%15r>3Xd@XGemkv2wEXaRASy4hjI?$PrP^8_ZLm{acA zn~3`jJ?tCgcWd^KT+S}aPdlp3vh!Gq^Mej)#~3{h)cEhwj#U6y0-ID;#*dgu;+z8Q@EzoGD=PnJcCdm z;TAv9Gcd)=HS<}uSC@=<5l)DC>E9E*DX<;vzp)Sv8GYyV@-rN3NPHICS0>-?w^(kx ziw|>=Dzi{a_2?p{GSgnu%Ll1*1+4TlEl1?*%XIkiQH;v-M@OEEks%kT1!JrrG~t4j z^=3Yv6fjM}f)aJP=pV6HQv8~1ybowyE>4N>-$}lsuvP|PpqYQ)PI#CrIj3ISB}z2r zTOq*MMFkCCxA)E}^dhc5_0!&^!ipywFy`U9u274!IfP41Y|(#XJl2jeKBE$NZC!Ny z@!Ou{!JJ3K9EV1J5`kukyXS*kHl>-=>=8tIrvp~6&+~?8m6>qpHelfME#~%PW&eF6 z&c%I5FfpRL1X|d@E&P+H{P%<~^eIDL|mV&*?-UQ0+^Y1NF1XUU_ft*jM7X# zFdVEItU)#!f{mGNX@3q@CS>n!ZLbZOF+(WYdT!raS^sUkpEe)aV;~+$oJdR?aaWok zGaMj~_q1W!vj-c@x;bWI($+7b|)szk{! zL29&%`UN;krM6|=MSj(42WQR4*F1w1gUH0v2^OpLha=^Lv#`o4k(5Kls7|lZ-;1*J z7caBCQa*(Kt4_R?!4F{q!ZQY3YQF(A5F%A0BMV^WP{GLA$ohNo-N@Py;9_QNXyXDf zcCt2jifkPK%&gxLQjQUj>mfuAJh_95dZR0}dOUaexwnS#RQDN{?gE)qwt{I{ht86d zK`YN%>X-JEN%g>Pp=dnMH~BUp7!4B`X%i;76)0`_E;SdNBQu5-aR^?m6xd=fjw;(Z zKi4=2>q*b4%?Q#z(m@`)op{^Xe9D%3*dRVEuYoEaZ0G}Xzknji$J{6KRUnzRTMO4# zws6D#qybcPSFkCxt5^y@;2Gogv|Eir7dG3A3Mnlo+p(MY!rgG6ebR?&MlPzX**V& zZ>?tKIJ_;OT-puoR_ZhSiE>bBsor0Ot9?sY$PXpH@xf=pPq-|_Oyk36C1GNEY&>$oCa61`w#bS5vWSr_Mk}- zy4en|vTN&vFiFURBB)EEO`FmiLc(yR_ihH<9z)cF?Wp!2Bu`m19O-4bM)u}gf_HCI zP0O?|a4Qz=^&uj(N*4X<>lXYn>Al~$bC0|S^)*JJPx?H(x9~9)XVnZaO&fZxtE|P6 zu}N;qfA(S`qY3v-G(`t5{xYb}7k2sX_yBa2%npO1p z+&Hq(3fuGPX_;z?5HKV}o<|ZE$n8jEvSCI!bt1#AVrdWy;n(`a;hybsgTJIihh(3x z5(AOR0WPKAL+8)5r+|Thlf9mS+xPq@Kv6(OURngeK+gzZX8D1Yd>0U_XTWTzeSlED zqxSI9GK4rTzu*!|_JGU^ox)VY%pxT~ZmtTP?Uv`Z%|vd@TnXIO@_Gw?-Jc*G57DE* zDJ;w60T`9E+#es^U!)8x+l@byqScom0CteicWmg5 zHfrEZx`$hhLi`P=-@M5pYRAN{dHqxfL$oOc%{oHd67^EaPM98Q4rQi$vKNIO& z3$kE3yd5@ozW>%rnPMsmsiH=ns{r!O?2`RcvbpG6Ov0lMpy+^UyV4@)u3^}QjvYOPjyk7A8E^?X~ zLdcmHA!ABv)>2JW4X=K^Fd=2mfu#ZBF$G+bzlVu}(NnVEkL$#r!4!a*@ed@16@a>| z_520yMW+gWf`$FX=Hi*L`V5G_lmua%-u%S@*(C1e+$!_LY7K;qyvSSIZMpF~NCeT% zip6pG4hLQF$*_}@-d-hP-rm^$=U~il2f3nE_7BPob+r}IVLj9yOiUhg_Bd4u!0=*5 zWArpc`>s{HJd_oyosvE(ynqo%K^sDAfb5z_9$H2iU-L*(2elgc#6q#_{FyMvk1A2T zn3fI6aj3{6jded_GQahroZBe8sa}o8Vx{5gl^+jTP8w2x+q`{q;Rb~qiA`$0o_Ci_ zSipL1Cddfvi)&cL#X5ynJPz~adWFmGjY*hA7s4Rx3H_J}UiN_g^2tN3)q2#1;7=0_ z@DT8G)XPY+f+w}SVfeQy;#Rq?hb%A!J196L!Ct0|zlh1zJ}4MJ&;g%^_=oijKmCFK zvoiW_fbagcl8r5ZkpaN;wr<_MOU44j?pib5<3p z*OFK>L_J&QiaRcJ5;BYFMUa` zaZe(Sx#F^ou=y1A$SpMB%Wm??0&k(0UhPGoIm2`pjAhj+?Xre=J$js942#*zs2i!t zwNwc)*J5aSG%z=AjZr){!jkvWDCxO#i0Z_y?uq3al4whinpVg}_ObCpI|R*mU6@8m z@6CNgh1724AH{2EW&_Li096-C*@gXfnGMlT6Zqv_r%JF7ohmx&7wYG930E&G6Nj=- z4}u}YXKW7wFYgGK@g(@cqTN|^M&VD(6(5MxEMwBU2N}Ug@i2?PV510y@=zh;-nZ)w zo$icH#JZ4f9<&9XFoSzaB0@kHp$fX@y=%3q!}%n5x>eMGs!5zcrVH*jC>Z2hN5D(T za{%!sXMus;AiT|~`8qZ93+u>wK*J4l8qU~6%I9Ch&Vn+|`T!f&KGZBmw{W zaC1a2Yvz_{h-01lOGOQ1vqZWuM|xoujDzP zLgttvdF91Sm0Iq^u;&H$5Z!x8^_ZzebQ;$^kicsa#So_xYvT}p+b4D(Uenm0((4F zF(^KT>mId_EUBIT>==u{6mQ-$1pwx5rHm`z#yX0UUG_DT_7aZ7lG(NKT}gGDxGsT z>FmHxky`iM-Q~>**Z7*55~%1Q(7t*H1>C)e@8cia8aYA4O4?nHU;Ze4HL_6}pE}Sx zaURoVf{n^?Fj4sNZhgvDT@ea4Ez{dr>?&`MJ5kWDizJn>i~TC%T&T}c#el^qX+sJ3 zi_ZSuIc5@MS2U)yH2jzM1)|iNSrzXs9>R`Q<%#G8IOJm)v7`ep;r`#?hZ_NvI8X4u z3d+pLCo6v-Aof6=F8zCgB5r14N(b~zjSK<8_GZ8msXyZgCid_0;Tb0btU!47q|%$6 zK4cL_&9X99YBz&XRe}l4>52KqnTiGSYI#D8dfglLa##khjwEjjwI}3EK5qgK=W$zz zZ^MuN3Ieod*k576IW}Om`$Y8;1mq;#=TVa*%}9fCp(De>&)4Gunj&bVvt+dFDehAX z&XqpF@YpApb#}(f>Ar&nTUcNVP?HuIz79wB$OP+C6MtEE)IWWGWfT31tZi^kd#HR; zU*E?Ew{)ET6jzzf*X|iRx+>(^d6Og*#5-)SZv|ThgZ;EoFmOV0-MxHRehmR-)_k$@ z<2j$*^r@hnzQ!Rf;Q4iHe;Jh$gC7@f7W(=c5TdPzPqK>!{qrI zVOy=T<1#upF6S+nm6dMrtWm8>|m0QYX*b3l30yxq(;w9_{8F7PIjP#pz3y zkcrwZVpx)*E`<nr{eNix2uheZzJsm(`g)l+oE)okRkc<}fy<8H9ptc|$Wu;-ASfxfX^=%rge z;&MhnPkFnTH|t(|2-}L^ESw#$BBv8A^dU%Bnt1Zfs=1QZh_~W%ui-{Pq@_mGhWYiC zanx(dBW?`ccv#x6<%1} zN%F1paIXPS>KtD&>P(nTCb4c$MC*OwW`7}-bwKO95xUm4){_A0OWs}Mjknvgd>W3k zYO!8POv0@pA~dbRQrG|*KknoF$sU&*|IVH;hG3xW*zTDWJwzFHJ>5|KSg}l4DzV(K z^kA(uTTIfcZ9$co`6lr_T6$hWfKdtU>y0iV8MGHm&*#dJm7DusQNgI^FAMT7g;jS; zCtWm;#S{D$t*hxb<9tA*c7aRe_eAqQ(NY1%^fvYYrXNAPw*2su&1!vu$-ktQ)#{yQ zpsNHSk?$LU0gdHUU8+!jg6y3GI~fO`<*~8D!?Bw&N;L4VYVil|)u|fx@Q9(@aXg4i z&4M|qV-Z~o+2lfp&4StR`HY2u+z(b%?1KS)Ao=Ge)=cZfxc#;KI0OL@su3GmS55-r z2bQ?;GSpNlXMG+r-9ye*1aZU5?t(l}CJB!(HuW6$JeMxsMFQ|MG6-y*zml76%9!K; z#QVYSS>gJH=$OFPmVH1MeD8AGvhtZkw9s+_>lN+C1>66MCHdvS>uERfuhJ zef^6&=Q~GVZderU8LAE{sx=#WB+ip8UMWu0BWl7Z4n>yhlf=cCnR(k zO9@otmC1i1OA?u^Wa7#>Klh$5@klg*19N?L_j!VW5}SxxL$@klT3plx2ecNYv|Fp% zvkGA1{a2CEdZE}C2o3`B8hAbMkBVS_x)@3}j(V1V#EVKchI(!Q#vh8UO&Ms#*pb@~ zRAE{TF)G10lH%1G4!alVC)j-th=Yj?xCTsYNl2v38`gD15aiD?kGwoEgNSv z&`1F*+G>Znt_5n-#9pSPek33t{Tu>ex|GEXIf$cXWTx!X25S^m9jzkRAptj{(3W{J z2X{)B2~}wlZcZSPW<{MEbGY`ZhjwXN?0hj@$(Y{bj%IK&g0Ovzt9dO06sLOA2;iZu z{hr$;Mn6TKcbnj0G}EWjL(xG)+$hlbUYJ0H(dv@0QRai_+2c|W)4_-n2}@0#?1$OA zn@eN2ixm0_^v4m;BeLq94b<>>Sl{)V(_En!76_zGw=94g-1XVkicUGOqp6Y1Qe!hU z2BQErx}L2GTg*PZ>BQ1Gp-PLhKDwT6yLolf{0#b?4>jyG?CBY!BfZwc<205CH>)s7 z@T;emf~tHV>51r0Mn$pX$TYk%K}b>Ar)ZUUMsiaiSyE7FUQO*9<$Ie47l3oJES049 z{Xo5&H-Vr+UugxM4GH7w=Hza|sy^y?T#y8@u9@wUAf)38s?`w#jz{WA#6+>rj*qe; z8TY)QaN9|kkcpH1y(mPExSsgUQQ)^a+YY_`jn{lGOvFy$L0i85PQw#6MyCF%tOPCvSGbwW~Zr*)a#O(73yT{EPp zW|?ULMtl0)Li1fpQ6{4X1MY0a#v+dc~bVKQ#PI4pQ3-jm6KSk+D{tM5fTizI{hn=z2o|&f=7>?AfXah zIufA?%at#YzjT3ww=LS}f#z7wE7~1CtQHsdt}}4o?xD0|4mA{9i_b=|_5QG(qYq1<`-< zfM!f@)`fUbceP_dRXf3NHZ3KHT+Oql980Gf$F@j&eT+94Z7uk+?FAs$?N01`dO$Sn zx%x29$Ooj%Zq3<1mtqlA^nx!q_*F-Yg%b%Hjw*$$NXt4yS;AH#bd~zO`*lUz^byLY zO)GgA)A`F1wo$6nGv*MuGt$t`IwTG? z9r1u<81o%)b9w~P4J8Qdfs=80c(d0n7eWxE#Z7jC&|YXq8Ki`M{ov*Wp8{OxC0tNc zSOz}X+DCv$K#Lah`;mwP%<}maX4s@T%2yyc&!U92+UCD#=NHHgW^53?A_>4K9$Z#g z)T1Zm8*Ll4Bm&SAGvId_5`E}gLLk_Va)M|}vcycQ>RnTty&DyF#rb62t-gvykAPu1 z;NS*`A&o$ALF;0F`v}^}r^puTIy}~)&t@@9>-R?n;^-*@VaD@415t=>eEwE@ddo@d zA$X%CYkLRup1pW2iqEGVLmB@#thCcAwr_dI8pA}QEqj(Vl(9@zU1L&J*EaY13*Wo# zH+F3>cSl)9{({v#v*aaVu5GLNcJ-R7jJvH7ssr=A| zNR*Or?R+AO>MTLw{!4!>f$zn$Q=(KLV04Q;BWm&q#;_&)aAkfFL+uV%@K#* zMWqxv5q4Yp=7om5)U;`Y%VxHuKGqKTqC(MQHosqJ^L#Y4uX zsR}bk&1khnG?Vq6tMa%XJsxsDJA1a!zk%D1#Wf1d&VGaZp3Y)MLSP_&Wd)WRda_n+ z*jUr1Wb>D3X+JT!R}}~$7!bnW%sxMLu>e$TES;>3p7ac`cMIr%vzP*y{>VPZes{t} zuCC!;?sqFsbH6hVj|mgblSE^MQAah2Q)9RLEGF^o4em7kg4drE6x5J(phQ5A+xhenHFWGoR2!Sf(T?-X$dG|Rw5*o#c4|{~*|Y)8~y zZ_~y2pxvOCu#H8cq3ZZQl3kcZ^g~Z^Op>uOnb(zhWMx3}J?o^ETPY=f@!+Z>% z&VF{{?x@+vhAD%bW%a6FH+ibTfd4sl0+9!d8;zW1*-E43+39{xXcuYg#TdXxIm$cx z!qIL-13_Q+6LB_afrH1FmDHoR%90w4^f`EMOgS$`N$c5X*qrSOP5Nea?W(WhK~EUc z1e>@VWWU9+&|@!9h=WcC<(s{hH)YCIkK4X3FWSs4%%L$yZ70>SnM*B%f`_8JJwUc$ zT+VY6Ie#y1+B|Ln2a;wC<5nkiwM;7U9`ILRa0L|AG8^a%mI8f2{@-)X|IaS`@B^6` zz6ZFyKtFJc^@ksr3wg+2pXgMose!o)c5v7|sE-uxgBe;o8gK(6!Ep+8eS#NdsO~%k z!gFCfj)Q1|zs{{*#xw{&VD$XqnnP=-jMf}$EeXuoH-?qD*uI!HuClNWk;yDggNn+v znG4ZqW4bX0F`=HWUb&-jSy_!s*+OT=jT*rv4;)WPm#{$sIS34Qp_&ydKazM1FIPkV z&i{j+X^{uCIHB2%Mq~($Q=}h{JWh`e&OXBfbhIB|*-#AfI6!1_JW3MTYxAbRDhyiMB;(^9Pd>tn zezWE@(H<5}!{zM^b$7Qk#Og?wdE@)x!ySlXq5dj#Dp5xqdpQm{Rrd?-=Kc#IuS^F< ztjLUdxr6!v4JX^4jKcK)Rlro_88L`;=SSm7<7Yidc2q&pA2 zGSfXaI5Z@uVqI_*6-@RdaM6a1^Gnh=#D#-ZpGma2t14eQ#9)cH_qDNoY+JWJ#ZzA4 zq51T9zQ$}GIBErp#rXx+C6!>snMaEEbZvoxUkzX0IrU3z&`hA~qAV5UZ>I*nSI?rf z-5JzmIm8ZPungrWS1!kuirJrXO>ynl4`qH;t%);=lWZndQbu!rCAjK-Ti8)c!2U*k z4lALc;}zwzOir}-?!o$l>`RLK!}l+RiK9I&-ecTfX_PUxP3mq*olu?XO4&e!5;i>t zGn8ugT2D~hlrihn5c{0ue^_{X)fdjWJrlxhN5FwnRLqP%Vm&&L8f1fe%XNW^w&1qH z_TYgW$jt3ZWeV+FrF~+&=?0(yxPDiA1AW9FIW7l)w2_{nk^PU(Vdfu-Kslz(`U4?yze|Q624OdAm_fzS zM7BXgu^@CB4(T&`0*UE@m-&Mer(P~M-dgr5IbSO`r89N*G#B<9YB3^0+letFsQFoX zbf+e*7oqqR0cj-I2KC`0B(q0+QhY#J;=zG82(^%f*WY(zE>1xmO;)6wShzGc;%a z)=pm=DGYWp6Ej3lS3m6=N|1!9zoECyt)a^XwCpoOaWx0lK#|T$%2(B(!BMbzkX#pV z3-5bEw8G8aa-RKv>X2yvi5~v5L*v(8>Z5WMyZ1ni&V>j9BK4a)J+*A;+3M+=S(-VT z0lj}kQyUipU>_~uM-@d|@~XoUJ96jxH4TfW*|7kYL{f)q4nHqk;7kn3(JBIo6qD)Z zLOEaD*?Q(7zyH)T-nUJyNA%UsR|T`>$fat*RKva1N=CFT8$yq8ciJccRC_fMfz&E_ zagcAzrYXkYuCqxDyBrb|sb!i8i#8iSgzMA1u&Z0bGSVp)m#BKimLFPkvz!-0rg(Zh zTncBPT$;D%$5dfhP@I77`O!T>9Ky<2PdxOXJBEaS35Tovz;!^VH9jxU{QW)%vUI`u z&P5hF=!V6Ma4qt2Y>4Ep3;xFi(~INf-bfv5rfbW|&(0s0TOg((d)%286hbmvQ6#w`S%XZiAaL-G%NyQh{b5w7?)a}Z7M zzkg&{nFif9c$kmzjKYEBAaKq(N+%HX;bu4aB}tN}i^u5MnMTFJ15~F*@oe)U7>*T! zoMgfQz3t^4>vEeb`BBCo==e6o_~

wsi{itHwwCvHg{@%fmA8Ma=?-f}@Om(DARr z#}%lo0?YZIF;~2y?KoXpc-(LKFE963(w)c0K*z^IR{%Ft+2ELCWi7pT2LlW`Iv#jY zU#`Z3PKQTbIdiSo8REvL2L}!Sn4*V|i?^Q^m_IZ&5T6a&wGmcX0M7V^3TFqA{go`ULIs8`0Q16IL5p&Kd;` zUo0{wLjyKn?Y*q7$dM*Q?cSR~`^y~tNOpq4L|V!C#1MAk*CC&w1EGuPAPr+Jgh)dp zMMz14-(Ts~weXjfMx;7#V=|~;RMX0N`}Ko8^JdcVTNb`I4U;x{r|8`fZ z->5mCHUK3T27XU{#;?0SLnz>=X2Od=zvXyJa>P_fj8{!s-?#L@T#RyRBFTs@HN=pT zA^#n0GJWt2@w1yAq-;4XXW~I_zqNg^<2P0`#5q(193N$bnJ;5{snvb)ca_J!K|R|R zf*UjD(QXv-8LNVYcZhn0Q4Sx%9y9myj-fW7-FG26*S68*_5|F5&0j+g>%->(XkXX^ z_jlq9veUQJ;L%~ezUJ7R!)zPUrq>GXpHr6fm>vo;KU zOrFab3I8NVz$>;}c>YTng8me=)d!$oehcjClKXvCAZKf254^p2DsB_9v9i(wR=o)U zv$4P{tRIY+Xk<=2XBwLVdsefbQ1JYj zvPSV;A&D~4nQJ}!p?|GpToh$kqB801G-u%%6QThbMc2x+P|j1~;<*07n_=}Aq;@D& zi_k_oMY77hOSN>}R@9aQYT`sFGeH6&cp-u>6oM}TKKPB%(rh-rLj=h2=Q))Yi*P z=(XMjHcst?MyMrfaF54pKcktc092Im;n?;nPH;knQ$?jqP?3_(6Sbt*z%J=Vr)CVp z2xsDoC2NtsJ&!T*Cn&1)R(0s7w z(-V4)-YJvf`6w61DB739~NI z`Nxn6;vp2UOWe7@C1D1Bv;zWcUs7XC&6$UeXFMgXh29ILijHBN-oMlynzKEKbpYiB z8dx7J`}^|p$Bo}>Cq1C~bOQ)j>sh)vni)6%eq1WGsYp2h#ijk6hNWpcH-c^$$<9Ft zM_e>IlTtc$`B-wv=1TKO(hpscwV(;pERT5+$|PC) zOO9|q>Z%f z;tJ~&w4p9K$Zi8?K-|K83Glxy*ti*DC13l(=vEtf`oS}X{6mHjuCooSO@Oz#O(wv97&7>@YUi-L9qadi(y^DO&Xv^!NShq4`)bRH(9~tcg2_y-ckh%PXm@cV$0-7U&XUOQ?3Bp*f9tlNM za&Q-?Sw;M>^<`v0+r_C3b-ama{a_Fw1c`j<-+%g~izX0O0?kb-%g^K26mudpVT_JT zI)IHW#_PZ;Py!FmSv=9CZ*;+2H2^CpnWKv<5OrVK%#1rVaG@I_i1uyDju`&|U|+m6 z+rf6#-DIortEgCXb+UBV~Sqz?rMM zrKBwIGMjKypwOnC2IT3-4RZ%^HaEA4@xcu>5NX|oA@UxzA3JSRrCOI?%gh`!9+#pjbwxB=-+gtA1IdZ{|(tX5%*L@!f zQMqephNm<$+qJcBuFXMHL+l&ikuslu6%SiI$F>Q9j0`KFDXaoR^%Z6u>zmqx9?S-Y*^r=*FJi^U8= z4j7F`ogMF$mz-MP3$3M3&vJrFv&l+VW_3bdqcK-!S7y;ALMLL{r9|ddKOcG@s(dtQ!Mz9_ z8Ns~!GTaD8`%=Rhv@T(#+{gp=9BhID3ATQS*CyHeYZ4dl1=foNNMrryZV(R3%nT+o ztWCS1YO!l(_F2>3GQ9$eGs~WF1V=h)92A^<vNd3|zZE(Zfh{ zQ}hI=(;E|IFZfT`c2*sQuoHY3J6_8)6vK&Oex+n$5<@$|o+1?VZJBIp;gsVee^X@9 zPKiw&mfO&KawsnQ5W1NU9}N~3VLK(9S@PHw_Z^+FhpEgReEQ@Y!iW#a|sNxLBa6BfmQ_g`fqc#zW)aZ zGRTp@)4M-KNMZ}PuS1S;mgpELCCS03XqxjSPRzZ~2vsU#DsOBm?%@zedJ? zovraJCBKdu_(_Q~(!WN@-_@P|hs^&}HNUol|D??F`M*Qi&whsgO2Ds8vOfu!fAQ}S z0OXbb+A{kCr2lRmo_Y{}yq;7E7}h_aHR_;;&@< z+IRPpj05a{{^fvV{8ks-0pD*2*Nmw4~be7jvzh~g+xSzLKGw#m*5;~ z*K!4|xP{t`jrrRryE9(2vcDnOWOm-ir{(7YSgLCQgT842*>0UOvuVlQp-KMEaw{Wh z%1nSIUq^q-}UYb;G#2m}+u-@>il;e_dC@d`Vf%jyR= z-cMoHS$-(uv@=|}*yY@eSN2>> $*\033[0m"; } +ok() { echo -e "\033[1;32m ✓ $*\033[0m"; } +warn() { echo -e "\033[1;33m ⚠ $*\033[0m"; } + +# Confirm +echo "" +echo " ╔════════════════════════════════════════════════════════╗" +echo " ║ WARNING: This will PERMANENTLY DELETE your ║" +echo " ║ ClickHouse Cloud service for the NYC Taxi lab. ║" +echo " ║ All data and the service will be destroyed. ║" +echo " ║ This cannot be undone. ║" +echo " ╚════════════════════════════════════════════════════════╝" +echo "" +echo " Service: ${CLICKHOUSE_HOST:-}" +echo "" +read -r -p " Type 'destroy' to confirm: " confirmation +if [[ "${confirmation}" != "destroy" ]]; then + echo " Teardown cancelled." + exit 0 +fi + +log "Stopping ClickHouse trip producer container (if running)..." +docker stop nyc_taxi_ch_producer 2>/dev/null || true +ok "Trip producer stopped (or was not running)." + +log "Stopping Apache Superset..." +if [[ -f "${SUPERSET_DIR}/docker-compose.yml" ]] && command -v docker >/dev/null 2>&1; then + cd "${SUPERSET_DIR}" + docker compose down -v 2>/dev/null || true + ok "Superset stopped and volumes removed." + cd "${SCRIPT_DIR}" +else + warn "Superset not running or docker not found." +fi + +log "Destroying ClickHouse Cloud infrastructure with Terraform..." +cd "${TERRAFORM_DIR}" +terraform destroy -auto-approve -input=false +ok "ClickHouse Cloud service destroyed." +cd "${SCRIPT_DIR}" + +# Clean up state file +if [[ -f "${SCRIPT_DIR}/.clickhouse_state" ]]; then + rm "${SCRIPT_DIR}/.clickhouse_state" + ok "Removed .clickhouse_state" +fi + +echo "" +echo "============================================================" +echo " Teardown complete. ClickHouse Cloud billing has stopped." +echo "" +echo " The local dbt project, queries, and scripts remain intact." +echo " Your Snowflake environment (Part 1) is unaffected." +echo "" +echo " To re-run the lab: source .env && ./setup.sh" +echo "============================================================" diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/main.tf b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/main.tf new file mode 100644 index 0000000..aa213fe --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/main.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.6" + required_providers { + clickhouse = { + source = "ClickHouse/clickhouse" + version = "~> 2.0" + } + } +} + +provider "clickhouse" { + organization_id = var.clickhouse_org_id + token_key = var.clickhouse_token_key + token_secret = var.clickhouse_token_secret +} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/outputs.tf b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/outputs.tf new file mode 100644 index 0000000..e1d153e --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/outputs.tf @@ -0,0 +1,19 @@ +output "clickhouse_host" { + description = "ClickHouse Cloud service hostname (HTTPS endpoint)" + value = [for ep in clickhouse_service.nyc_taxi.endpoints : ep.host if ep.protocol == "https"][0] +} + +output "clickhouse_port" { + description = "ClickHouse Cloud HTTPS port (always 8443 for Cloud services)" + value = 8443 +} + +output "service_id" { + description = "ClickHouse Cloud service ID" + value = clickhouse_service.nyc_taxi.id +} + +output "service_name" { + description = "ClickHouse Cloud service name" + value = clickhouse_service.nyc_taxi.name +} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/service.tf b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/service.tf new file mode 100644 index 0000000..48963e8 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/service.tf @@ -0,0 +1,21 @@ +resource "clickhouse_service" "nyc_taxi" { + name = "nyc-taxi-lab-${var.cohort}" + cloud_provider = var.cloud_provider + region = var.region + # tier is only required for legacy ClickHouse Cloud organizations. + # Omit it for organizations on the new ClickHouse Cloud tiers (the default for all new sign-ups). + + idle_scaling = true + idle_timeout_minutes = 15 + + password = var.clickhouse_password + + # Lab-only: open access avoids per-partner IP whitelisting friction. + # Restrict to your office CIDR for production use. + ip_access = [ + { + source = "0.0.0.0/0" + description = "lab-open-access" + } + ] +} diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/variables.tf b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/variables.tf new file mode 100644 index 0000000..aa346c0 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/terraform/variables.tf @@ -0,0 +1,40 @@ +variable "clickhouse_org_id" { + description = "ClickHouse Cloud organization ID" + type = string +} + +variable "clickhouse_token_key" { + description = "ClickHouse Cloud API token key" + type = string + sensitive = true +} + +variable "clickhouse_token_secret" { + description = "ClickHouse Cloud API token secret" + type = string + sensitive = true +} + +variable "clickhouse_password" { + description = "Password for the ClickHouse service default user" + type = string + sensitive = true +} + +variable "cohort" { + description = "Lab cohort identifier used in resource naming" + type = string + default = "fy27-q1" +} + +variable "cloud_provider" { + description = "Cloud provider for ClickHouse Cloud service" + type = string + default = "aws" +} + +variable "region" { + description = "Region for ClickHouse Cloud service" + type = string + default = "us-east-1" +} diff --git a/workshop_public/snowflake_migration_lab/04-evaluation/assessment.md b/workshop_public/snowflake_migration_lab/04-evaluation/assessment.md new file mode 100644 index 0000000..1c588f0 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/04-evaluation/assessment.md @@ -0,0 +1,393 @@ +# ClickHouse Migration Proficiency Assessment + +**Partner Name:** _______________________________________________ + +**Company:** ___________________________________________________ + +**Date:** ______________________________________________________ + +**ClickHouse Solutions Architect:** _____________________________ + +**Lab Completion Date (Parts 1–3):** ____________________________ + +--- + +> **Instructions** +> - This is an open-book assessment — you may reference your `migration-plan.md`, benchmark results, and lab READMEs. +> - Section A: Write the letter of your answer (A, B, C, or D) on the `Your answer:` line. +> - Section B: Write your answers in the space provided. Aim for 3–6 sentences per sub-question. +> - When done, complete the submission checklist at the bottom and email to your SA. + +--- + +## Section A — Multiple Choice (80 points) + +*20 questions × 4 points each. No partial credit.* + +--- + +### Part 1: Snowflake Workload Understanding + +**Q1.** In the NYC Taxi lab, `trip_metadata` is stored as a `VARIANT` column in Snowflake containing nested JSON (driver rating, vehicle type, etc.). After migration to ClickHouse, which representation is recommended? + +- A. `Map(String, String)` — enforces type safety on keys and values +- B. `String` column, with `JSONExtract*` functions applied at query time +- C. `Tuple(driver_rating Float32, vehicle_type String, …)` — pre-defined schema +- D. `JSON` — ClickHouse has a native `JSON` type; use it directly as a drop-in replacement for `VARIANT` + +**Your answer:** ___ + +--- + +**Q2.** Your Snowflake pipeline uses this upsert pattern to keep `FACT_TRIPS` up to date when trips are corrected: + +```sql +MERGE INTO ANALYTICS.FACT_TRIPS AS target +USING (SELECT * FROM STAGING.STG_TRIPS + WHERE PICKUP_DATETIME > DATEADD('hour', -1, CURRENT_TIMESTAMP())) AS source +ON target.TRIP_ID = source.TRIP_ID +WHEN MATCHED THEN UPDATE SET + TOTAL_AMOUNT = source.TOTAL_AMOUNT, + UPDATED_AT = source.UPDATED_AT +WHEN NOT MATCHED THEN INSERT VALUES (source.*) +``` + +Why can this not be directly ported to ClickHouse? + +- A. ClickHouse supports `MERGE INTO` but requires the target table to use `ReplacingMergeTree` +- B. ClickHouse has no `MERGE INTO` statement — upserts are handled by `ReplacingMergeTree` engine combined with dbt's `delete_insert` incremental strategy +- C. ClickHouse's `MERGE INTO` requires a `PARTITION BY` clause to identify the target partition range +- D. `MERGE INTO` is only supported on ClickHouse's `AggregatingMergeTree` engine + +**Your answer:** ___ + +--- + +**Q3.** Snowflake's `LATERAL FLATTEN(input => trip_tags)` expands an ARRAY column into one row per element. The ClickHouse equivalent syntax is: + +- A. `UNNEST(trip_tags)` +- B. `ARRAY JOIN trip_tags` +- C. `EXPLODE(trip_tags)` +- D. `GROUP BY … WITH ROLLUP` + +**Your answer:** ___ + +--- + +**Q4.** Your Snowflake pipeline uses: + +```sql +MERGE INTO fact_trips USING staging +ON fact_trips.trip_id = staging.trip_id +WHEN MATCHED THEN UPDATE SET … +WHEN NOT MATCHED THEN INSERT … +``` + +What is the idiomatic ClickHouse approach for this upsert pattern? + +- A. Use `ALTER TABLE … UPDATE` for matched rows and `INSERT INTO` for new rows +- B. Use `INSERT INTO fact_trips … ON CONFLICT DO UPDATE` +- C. Use `ReplacingMergeTree(updated_at)` and always INSERT the full row — ClickHouse deduplicates on compaction +- D. Use `MERGE` — ClickHouse supports MERGE DML syntax since v23.5 + +**Your answer:** ___ + +--- + +**Q5.** In the Snowflake lab (Part 1), a **Stream** captures CDC changes on `TRIPS_RAW` and a **Task** runs every hour to apply them to aggregation tables. In the ClickHouse architecture built in Part 3, what replaces this scheduled refresh pattern? + +- A. A ClickPipes CDC connector on `trips_raw` +- B. A Refreshable Materialized View with `REFRESH EVERY 3 MINUTE` +- C. A standard Materialized View that triggers automatically on each INSERT +- D. A dbt scheduled run via GitHub Actions cron + +**Your answer:** ___ + +--- + +### Part 2: Architecture & Design Decisions + +**Q6.** You are designing a ClickHouse table to receive bulk INSERT batches from a migration script. The script may be retried mid-run, causing some rows to be re-inserted with the same primary key. Which engine makes retries idempotent? + +- A. `MergeTree()` — duplicates pile up but can be filtered at query time +- B. `ReplacingMergeTree(_synced_at)` — later INSERT of the same primary key wins via the version column +- C. `AggregatingMergeTree()` — aggregates are commutative so duplicates cancel out +- D. `CollapsingMergeTree(sign)` — collapses duplicate rows using a +1/−1 sign column + +**Your answer:** ___ + +--- + +**Q7.** `AggregatingMergeTree` is the correct engine choice when: + +- A. You need to deduplicate rows by primary key using a version column +- B. You need fast point lookups by primary key with no aggregation +- C. You are pre-computing partial aggregates to be queried with `*Merge` combinators (e.g., `sumMerge`, `countMerge`) +- D. You need to retain the full history of row changes over time + +**Your answer:** ___ + +--- + +**Q8.** Which of the following `ORDER BY` choices is most likely to **hurt** analytical query performance on `fact_trips`? + +- A. `ORDER BY (pickup_at)` +- B. `ORDER BY (toStartOfMonth(pickup_at), pickup_at, trip_id)` +- C. `ORDER BY (trip_id, pickup_at)` +- D. `ORDER BY (pickup_location_id, pickup_at)` + +**Your answer:** ___ + +--- + +**Q9.** A query on `analytics.fact_trips` returns duplicate `trip_id` values. You add `FINAL` to the query and duplicates disappear. What is the trade-off of using `FINAL`? + +- A. `FINAL` permanently deduplicates the table but requires a full table lock during execution +- B. `FINAL` forces an in-memory merge at query time, increasing latency — without it, duplicates can reappear until background merges complete +- C. `FINAL` only deduplicates within a single data part, not across parts from different INSERTs +- D. `FINAL` is only supported on `AggregatingMergeTree` tables, not `ReplacingMergeTree` + +**Your answer:** ___ + +--- + +**Q10.** In the Part 3 dbt pipeline, `fact_trips` uses the `delete_insert` incremental strategy. What does this strategy do on each dbt run? + +- A. Runs `DELETE FROM fact_trips WHERE …` to remove the overlapping range, then `INSERT INTO fact_trips SELECT … WHERE …` with fresh rows +- B. Uses `REPLACE INTO` which atomically deletes matching rows and inserts the new version in one statement +- C. Drops and recreates the entire table, then re-inserts all rows from scratch +- D. Marks stale rows with a `_deleted = true` flag and inserts the new version alongside them + +**Your answer:** ___ + +--- + +**Q11.** `fact_trips` has `ORDER BY (toStartOfMonth(pickup_at), pickup_at, trip_id)`. Which query benefits most from this sort key through granule-level data skipping? + +- A. `SELECT count() FROM fact_trips WHERE trip_id = 'abc-123'` +- B. `SELECT sum(fare_amount_usd) FROM fact_trips WHERE pickup_at BETWEEN '2025-01-01' AND '2025-03-31'` +- C. `SELECT avg(tip_amount_usd) FROM fact_trips WHERE vendor_id = 2` +- D. `SELECT * FROM fact_trips ORDER BY total_amount_usd DESC LIMIT 10` + +**Your answer:** ___ + +--- + +**Q12.** On `trips_raw`, the `pickup_at` column is a `DateTime` that increases roughly monotonically over time. Which storage codec best compresses it? + +- A. `CODEC(ZSTD(1))` — general-purpose lossless compression +- B. `CODEC(LZ4)` — optimised for fast decompression on hot queries +- C. `CODEC(Delta, ZSTD(1))` — encodes the small differences between consecutive values, then compresses +- D. `CODEC(Gorilla, ZSTD(1))` — designed for floating-point time series data + +**Your answer:** ___ + +--- + +### Part 3: Migration Execution & Validation + +**Q13.** Your Snowflake pipeline uses `HOURLY_AGG_TASK`, a Scheduled Task that runs a `MERGE INTO AGG_HOURLY_ZONE_TRIPS` statement every hour. What is the recommended ClickHouse equivalent for recalculating hourly aggregates on a schedule? + +- A. A standard Materialized View on `trips_raw` — it updates `agg_hourly_zone_trips` automatically on every INSERT +- B. A Refreshable Materialized View with `REFRESH EVERY 1 HOUR` — it re-executes a full SELECT and atomically replaces the result set on each cycle +- C. An AggregatingMergeTree table — partial aggregate states merge automatically in the background without a scheduler +- D. A dbt `table` model with a `+post-hook: "OPTIMIZE TABLE agg_hourly_zone_trips FINAL"` to trigger compaction after each run + +**Your answer:** ___ + +--- + +**Q14.** Your Snowflake `FACT_TRIPS` table is clustered on `PICKUP_AT::DATE`. You migrate to ClickHouse with `ORDER BY (toStartOfMonth(pickup_at), pickup_at, trip_id)`. A colleague asks why you added `toStartOfMonth(pickup_at)` as the leading key rather than using `pickup_at` alone. The correct explanation is: + +- A. ClickHouse builds a dense index with one entry per distinct value in the first `ORDER BY` column — a raw `DateTime` with millions of distinct timestamps would create an index too large to fit in RAM +- B. ClickHouse's sparse primary index stores one entry per 8,192 rows; a month prefix enables the index to skip entire months of data on monthly aggregation queries, while `pickup_at` alone provides only row-level granularity within each 8,192-row block +- C. `toStartOfMonth()` is required by `ReplacingMergeTree` — version column deduplication only works correctly within month boundaries +- D. A raw `DateTime` column cannot be the first key in `ORDER BY` in ClickHouse — it must be wrapped in a date-truncation function + +**Your answer:** ___ + +--- + +**Q15.** Your first benchmark run shows ClickHouse Q1 (date-range aggregation) at 1.8 seconds. You run the exact same query immediately afterward and it completes in 0.4 seconds. The most likely explanation is: + +- A. ClickHouse background merges completed between runs, improving sort order and reducing scan size +- B. The OS page cache warmed the compressed data files on the first run; the second run is served from RAM +- C. The `mv_hourly_revenue` Refreshable Materialized View refreshed between the two runs +- D. ClickHouse's query result cache (`use_query_cache`) is enabled by default and returned the pre-computed result on the second execution + +**Your answer:** ___ + +--- + +**Q16.** You insert 100 rows into `trips_raw` where all 100 share the same `trip_id` (intentional duplicate test). You immediately run `SELECT count() FROM default.trips_raw` and see 100, not 1. Why? + +- A. `ReplacingMergeTree` requires a `SELECT … FINAL` even after compaction has occurred +- B. The rows have different `_synced_at` values so ClickHouse considers them distinct and keeps all 100 +- C. Deduplication in `ReplacingMergeTree` happens during background part merges, which are asynchronous — the parts have not merged yet +- D. You must run `OPTIMIZE TABLE trips_raw FINAL` before row counts become accurate + +**Your answer:** ___ + +--- + +**Q17.** A customer needs to migrate 500 million rows from Snowflake to ClickHouse Cloud as quickly as possible. Which approach gives the highest throughput and is most commonly used in production migrations? + +- A. A Python script reading from Snowflake's cursor API in batches and writing via `clickhouse-connect` +- B. Export Snowflake data to S3 as Parquet files using `COPY INTO @stage`, then ingest with `INSERT INTO ... SELECT * FROM s3('s3://...', 'Parquet')` +- C. Use Snowflake's JDBC driver to stream rows directly to ClickHouse's HTTP interface without intermediate storage +- D. Use `dbt run --full-refresh` pointed at both Snowflake and ClickHouse simultaneously to synchronize the tables + +**Your answer:** ___ + +--- + +**Q18.** A data analyst queries `analytics.fact_trips` (no `FINAL`) and gets 49,998,201 rows. Ten minutes later, with no new inserts, the same query returns 49,998,198 rows. `SELECT COUNT() FROM analytics.fact_trips FINAL` consistently returns 49,998,198. What explains the initial higher count? + +- A. ClickHouse Cloud replicates data across availability zones — the two queries hit different replicas with different replication lag +- B. A background part merge completed between the two queries, deduplicating 3 rows that shared the same `trip_id` across separate unmerged parts — the first query counted all copies before the merge ran +- C. The OS page cache returned stale results from the first query's scan of an older data snapshot +- D. The `mv_hourly_revenue` Refreshable Materialized View deleted source rows from `fact_trips` during its refresh cycle + +**Your answer:** ___ + +--- + +**Q19.** The CH — Capabilities Showcase dashboard includes a chart comparing `uniqHLL12(trip_id)` with `uniq(trip_id)`. What is the key trade-off between them? + +- A. `uniqHLL12` is exact but slower; `uniq` uses HyperLogLog and is an approximation +- B. `uniqHLL12` uses HyperLogLog (~1–2% error, ~16 KB memory); `uniq` is more precise but uses significantly more memory +- C. Both use HyperLogLog, but `uniqHLL12` uses 12 registers while `uniq` uses 64 — making `uniq` more accurate +- D. `uniqHLL12` only works on `String` columns; `uniq` works on any data type + +**Your answer:** ___ + +--- + +**Q20.** The CH — Capabilities Showcase dashboard uses `dictGet('analytics.taxi_zones_dict', 'zone', toUInt16(pickup_location_id))` instead of a `JOIN` on the zone dimension table. Why does this outperform a JOIN? + +- A. ClickHouse loads dictionaries into GPU memory for hardware-accelerated lookups +- B. The dictionary is materialised as a hash table in RAM — each lookup is O(1) with no disk I/O, unlike a JOIN that probes data parts on disk +- C. `dictGet` is processed by ClickHouse's vectorised execution engine, while JOIN operations cannot be vectorised +- D. The dictionary is stored in a ZooKeeper node, making it available across all replicas without replication lag + +**Your answer:** ___ + +--- + +## Section B — Open Questions (20 points) + +*4 questions × 5 points each. Write your answers in the space provided.* +*This is open-book — reasoning and depth of explanation matter more than exact wording.* + +--- + +**Open Q1 — Engine Selection for a New Scenario** *(5 pts)* + +A prospective customer has a `sessions` table that tracks user login sessions. Each row has `(session_id UUID, user_id UInt64, started_at DateTime, ended_at Nullable(DateTime), event_type Enum('active','completed','expired'))`. Sessions are frequently updated — a session recorded as `active` will later be updated to `completed` or `expired`. Multiple services write updates simultaneously. + +**1a.** Which MergeTree engine family would you recommend, and why? *(2 pts)* + +*Your answer:* + +--- + +**1b.** What would you use as the `ORDER BY` for this table, and why? *(2 pts)* + +*Your answer:* + +--- + +**1c.** What query-time behaviour must you warn the customer about, and how do you address it? *(1 pt)* + +*Your answer:* + +--- + +**Open Q2 — SQL Translation Challenge** *(5 pts)* + +Translate the following Snowflake SQL to valid ClickHouse SQL. After your translation, briefly explain each change you made and why. + +> **Constraint:** Write the equivalent **without `QUALIFY`** — use a subquery instead. This lab treats `QUALIFY` as a portability gap and uses the subquery form throughout, so that the pattern works identically across all SQL engines. + +```sql +-- Snowflake: top-earning vendor per borough in the last 30 days +SELECT + tz.borough, + t.vendor_id, + SUM(t.fare_amount) AS total_fare, + ROW_NUMBER() OVER (PARTITION BY tz.borough ORDER BY SUM(t.fare_amount) DESC) AS rank +FROM trips_raw t +JOIN taxi_zones tz ON t.pickup_location_id = tz.location_id +WHERE t.pickup_datetime >= DATEADD('day', -30, CURRENT_TIMESTAMP()) + AND t.fare_amount > 0 +GROUP BY tz.borough, t.vendor_id +QUALIFY rank = 1 +``` + +*Your ClickHouse translation:* + +```sql +-- ClickHouse translation: + +``` + +*Explanation of changes:* + +--- + +**Open Q3 — Reflection: Biggest Conceptual Shift** *(5 pts)* + +Based on completing Parts 1–3, describe **one concept** that was most different from how you expected ClickHouse to work, compared to Snowflake. + +**3a.** What did you expect, and what does ClickHouse actually do? *(2 pts)* + +*Your answer:* + +--- + +**3b.** Why is ClickHouse designed this way — what problem does this design solve? *(2 pts)* + +*Your answer:* + +--- + +**3c.** How would you explain this difference to a customer evaluating ClickHouse for the first time? *(1 pt)* + +*Your answer:* + +--- + +**Open Q4 — Real-World Application** *(5 pts)* + +Identify a customer opportunity or internal use case — real or hypothetical — where the Snowflake → ClickHouse migration pattern from this lab could apply. + +**4a.** Describe the customer's workload: what data, what query patterns, and what is their pain point with Snowflake? *(1 pt)* + +*Your answer:* + +--- + +**4b.** Which ClickHouse engine(s) would you recommend, and what would the key `ORDER BY` column(s) be? Justify your choices based on the query patterns. *(2 pts)* + +*Your answer:* + +--- + +**4c.** Identify one specific technical risk or challenge in this migration and describe how you would mitigate it. *(2 pts)* + +*Your answer:* + +--- + +## Submission Checklist + +Before emailing your completed assessment to your ClickHouse SA, confirm: + +- [ ] All 20 MCQ answers filled in (A, B, C, or D on each `Your answer:` line) +- [ ] All 4 open questions answered with substantive responses +- [ ] `migration-plan.md` from Part 2 attached (or pasted inline) +- [ ] `benchmark_results_*.csv` from Part 3 attached +- [ ] Your name, company, and SA name are filled in at the top of this file + +**Submit to:** your ClickHouse Solutions Architect via email. +**Subject line:** `[ClickHouse Migration Badge] — Assessment Submission` diff --git a/workshop_public/snowflake_migration_lab/README.md b/workshop_public/snowflake_migration_lab/README.md new file mode 100644 index 0000000..1b59897 --- /dev/null +++ b/workshop_public/snowflake_migration_lab/README.md @@ -0,0 +1,38 @@ +# NYC Taxi Snowflake Migration Lab — lab artifacts + +This directory holds the scripts, Terraform, dbt projects, producers, and Superset assets +you run during the ClickHouse Snowflake Migration Lab. It is not the instructions. + +## Start here + +The playbook lives at **https://workshop.demohouse.cloud/docs/snowflake-migration**. + +Begin with module 00, which covers the toolchain, the two cloud trial accounts, and the +virtualenvs. Each module tells you which files here to run and in what order. Working +through this directory without the playbook will not go well: the order matters, and +several steps depend on state an earlier module created. + +## What is here + +| Directory | What it is | +|---|---| +| `01-setup-snowflake/` | Terraform, SQL, dbt project, trip producer, and Superset stack for the Snowflake source environment | +| `02-plan-and-design/` | Profiling scripts, five worksheets you fill in, and `migration-plan.md` | +| `03-migrate-to-clickhouse/` | ClickHouse Cloud Terraform, the migration script, the ClickHouse dbt project, benchmark and cutover scripts | +| `04-evaluation/` | `assessment.md`, the blank assessment template | + +The worksheets, `migration-plan.md`, and `assessment.md` are yours to edit. Everything +else you run as-is. + +## Credentials + +Copy the `.example` templates and fill in your own values: + +- `01-setup-snowflake/.env.example` and `03-migrate-to-clickhouse/.env.example` +- `01-setup-snowflake/terraform/terraform.tfvars.example` +- `01-setup-snowflake/dbt/nyc_taxi_dbt/profiles.yml.example` and + `03-migrate-to-clickhouse/dbt/nyc_taxi_dbt_ch/profiles.yml.example` + +**Never commit the filled-in files.** `.env`, `*.tfvars`, `profiles.yml`, and Terraform +state files hold real credentials and are gitignored here for that reason. Terraform state +in particular stores secrets in plaintext. From 7abb48197b911b31f4e4080000648c9cd337937c Mon Sep 17 00:00:00 2001 From: sishuo Date: Mon, 3 Aug 2026 17:23:16 +0800 Subject: [PATCH 2/2] Drop the worksheet copies the hosted playbook replaced, and fix the dead host The five design worksheets in 02-plan-and-design/worksheets/ are now interactive pages on the workshop site, which checks each answer as the learner picks it and keeps their work in browser storage. Every answer and explanation on those pages traces back to these files' answer keys, so nothing is lost by deleting them -- including the two exercises that have no direct on-site equivalent: worksheet 03's "Non-Obvious Translation Decisions" and worksheet 04's "Risk Register" both became reflection questions, and 04's register is also written out in full in the site's worked-example reference. migration-plan.md and assessment.md stay. Both are still files the learner edits on disk, both are referenced by full path from the playbook, and module 03's setup.sh reads the plan's completion checklist. Removing the files left seven pointers dangling, and every URL they would have been rewritten to was dead anyway: workshop.demohouse.cloud 404s, because the site moved to labs.demohouse.cloud and the old name resolves to a separate, older deployment. Both setup.sh files and 01_profile_snowflake.sh are synced from the upstream copies that had already fixed this, and differed only in these lines. The two READMEs are hand-edited -- they have no upstream counterpart. Also fixes the same dead host in RTA-mini-workshop/README.md, which had two references to it. All nine surviving labs.demohouse.cloud URLs return 200 and the three touched scripts pass bash -n. Co-Authored-By: Claude Opus 5 (1M context) --- workshop_public/RTA-mini-workshop/README.md | 4 +- .../01-setup-snowflake/setup.sh | 2 +- .../scripts/01_profile_snowflake.sh | 17 +- .../01_mergetree_engine_selection.md | 120 ----------- .../worksheets/02_sort_key_design.md | 157 -------------- .../worksheets/03_schema_translation.md | 204 ------------------ .../worksheets/04_migration_wave_plan.md | 183 ---------------- .../worksheets/05_dbt_model_design.md | 191 ---------------- .../03-migrate-to-clickhouse/setup.sh | 4 +- .../snowflake_migration_lab/README.md | 10 +- 10 files changed, 20 insertions(+), 872 deletions(-) delete mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md delete mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md delete mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md delete mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md delete mode 100644 workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md diff --git a/workshop_public/RTA-mini-workshop/README.md b/workshop_public/RTA-mini-workshop/README.md index 982b120..4cbb3b3 100644 --- a/workshop_public/RTA-mini-workshop/README.md +++ b/workshop_public/RTA-mini-workshop/README.md @@ -12,7 +12,7 @@ there and only the **dashboard** (a separate service) lives here: | Piece | Location | Served at | |---|---|---| -| Participant guide (MDX docs tree) | `site/content/docs/rta-mini/` | `https://workshop.demohouse.cloud/docs/rta-mini` | +| Participant guide (MDX docs tree) | `site/content/docs/rta-mini/` | `https://labs.demohouse.cloud/docs/rta-mini` | | Optional live dashboard | `dashboard/` (this folder) | run locally with Docker — see `dashboard/README.md` | The guide is linked from the workshops hub (the app's `/` route). @@ -32,7 +32,7 @@ https://partner-workshop.s3.ap-southeast-1.amazonaws.com/fx/ticks.parquet > `ListBucket`). To host your own copy, upload `fx/ticks.parquet` to a public > bucket and update the two references in the guide > (`site/content/docs/rta-mini/learner/load-data.mdx`, served at -> `https://workshop.demohouse.cloud/docs/rta-mini`). +> `https://labs.demohouse.cloud/docs/rta-mini`). ## Prerequisites (participants) diff --git a/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh b/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh index 0f5b939..3dc1b6e 100755 --- a/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh +++ b/workshop_public/snowflake_migration_lab/01-setup-snowflake/setup.sh @@ -516,7 +516,7 @@ echo " Next steps:" echo " 1. Open the Snowflake UI and explore NYC_TAXI_DB" echo " 2. Run the 7 queries in queries/ — understand each migration challenge" echo " 3. Review dbt models in dbt/nyc_taxi_dbt/models/" -echo " 4. Complete the worksheets in 02-plan-and-design/worksheets/" +echo " 4. Complete the worksheets at https://labs.demohouse.cloud/docs/snowflake-migration/learner/02-plan-and-design" echo " 5. Get SA sign-off, then proceed to 02-migrate-to-clickhouse/" echo "" echo " Cost: ~\$47/partner/day (auto-suspend = \$0 when idle)" diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh b/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh index 4805906..7f03a8b 100755 --- a/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh +++ b/workshop_public/snowflake_migration_lab/02-plan-and-design/scripts/01_profile_snowflake.sh @@ -304,7 +304,7 @@ SELECT table_schema, table_name, 'Contains QUALIFY clause' AS gap_type, - 'Rewrite as subquery — see worksheets/03_schema_translation.md' AS remediation + 'Rewrite as subquery — see https://labs.demohouse.cloud/docs/snowflake-migration/learner/worksheets/03-schema-translation' AS remediation FROM NYC_TAXI_DB.INFORMATION_SCHEMA.VIEWS WHERE UPPER(view_definition) LIKE '%QUALIFY%' ORDER BY table_schema, table_name; @@ -317,7 +317,7 @@ cat >> "${OUTPUT_FILE}" <<'MD' > Additionally, Q3 in `01-setup-snowflake/queries/` uses QUALIFY explicitly. > ClickHouse has no QUALIFY — rewrite as subquery wrapping ROW_NUMBER(). -> See: https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Gap 1. +> See: https://labs.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Gap 1. ### Gap 2: VARIANT Columns @@ -368,7 +368,7 @@ Affected objects (based on known lab setup): - `HOURLY_AGG_TASK` — uses MERGE INTO AGG_HOURLY_ZONE_TRIPS → rewrite as dbt incremental model - `CDC_CONSUME_TASK` — uses MERGE INTO from stream → retired after producer cutover; live writes go directly to ClickHouse -See: https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Gap 4. +See: https://labs.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Gap 4. ### Gap 5: Snowflake Tasks @@ -382,7 +382,7 @@ Replacements: ### Gap 6: Date Function Differences Minor syntax differences — all mechanical substitutions. -See the full translation table at https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Section 2, Gap 6. +See the full translation table at https://labs.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse Section 2, Gap 6. Key substitutions needed in this workload: - `DATE_TRUNC('hour', pickup_at)` → `toStartOfHour(pickup_at)` @@ -416,9 +416,10 @@ echo "" echo " Next steps:" echo " 1. Review profile_report.md" echo " 2. Work through the worksheets:" -echo " worksheets/01_mergetree_engine_selection.md" -echo " worksheets/02_sort_key_design.md" -echo " worksheets/03_schema_translation.md" -echo " worksheets/04_migration_wave_plan.md" +echo " https://labs.demohouse.cloud/docs/snowflake-migration/learner/worksheets/01-engine-selection" +echo " https://labs.demohouse.cloud/docs/snowflake-migration/learner/worksheets/02-sort-key-design" +echo " https://labs.demohouse.cloud/docs/snowflake-migration/learner/worksheets/03-schema-translation" +echo " https://labs.demohouse.cloud/docs/snowflake-migration/learner/worksheets/04-migration-wave-plan" +echo " https://labs.demohouse.cloud/docs/snowflake-migration/learner/worksheets/05-dbt-model-design" echo " 3. Fill in migration-plan.md" echo -e "${GREEN}${BOLD}════════════════════════════════════════════${RESET}" diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md deleted file mode 100644 index 6dbbb09..0000000 --- a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/01_mergetree_engine_selection.md +++ /dev/null @@ -1,120 +0,0 @@ -# Worksheet 1: MergeTree Engine Selection - -**Time estimate:** 15–20 minutes -**Reference:** [MergeTree Engines](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/mergetree-engines) - ---- - -## Concept - -In Snowflake, you create a table and Snowflake decides how to store it. In ClickHouse, you choose a storage engine — and this choice determines correctness, not just performance. - -The three engines you need for this lab: - -**MergeTree** — The base engine. Data is stored in sorted columnar files. No deduplication. Use this when the table is insert-only or when your pipeline manages updates externally (e.g., full reload on every dbt run). - -**ReplacingMergeTree(version_col)** — Extends MergeTree with background deduplication. When rows with the same `ORDER BY` key exist in multiple parts, only the row with the highest `version_col` value is kept after a merge. Use this when rows can be updated and you have a column that increases monotonically on every update (e.g., `updated_at` timestamp). - -> **Critical gotcha:** Deduplication is asynchronous. Until ClickHouse runs a background merge, both the old and new version of a row coexist. Always use `SELECT ... FINAL` on ReplacingMergeTree tables to force deduplication at query time. - -**AggregatingMergeTree** — Extends MergeTree with partial aggregate state merging. Use this when the table stores combinable aggregate states (e.g., HyperLogLog sketches, quantile digests) and you need background aggregation. Not needed for the NYC Taxi lab — the aggregation tables are rebuilt by dbt, not accumulated. - -### Decision Tree - -``` -Does the table receive UPDATE or DELETE operations? -│ -├── No (insert-only) -│ └─► MergeTree() -│ -└── Yes - ├── Is there a timestamp/version column that increases on every update? - │ ├── Yes → ReplacingMergeTree(version_col) - │ └── No (e.g., full-reload dimension tables) - │ └─► MergeTree() — dbt handles upsert via atomic table swap (full rebuild) - │ - └── Does the table store partial aggregate states (AggregateFunction types)? - └── Yes → AggregatingMergeTree() -``` - -### Staging Models Are Always Views - -In dbt + ClickHouse, staging models should be materialized as **views**, not tables. Views have zero storage cost and are always fresh — they are just saved SQL, not physical objects. - -The engine selection exercise below covers **analytics-layer dbt models** (fact tables, aggregates, dimensions) and the ClickHouse Materialized View. It does not include `trips_raw` or staging models: - -- **`trips_raw`** is a base ClickHouse table created directly by the migration script (`scripts/02_migrate_trips.py`) — not a dbt model. It uses `ReplacingMergeTree(_synced_at)` because the migration script may retry a batch and re-insert the same `trip_id`. After cutover, the live producer can also retry on transient failures; `_synced_at DateTime DEFAULT now()` ensures the most recent write wins. -- **`stg_trips`** is a dbt **view** on top of `trips_raw`. It applies `SELECT ... FROM trips_raw FINAL` to resolve any unmerged duplicates before the data reaches downstream analytics models. Deduplication responsibility belongs here — not in a ReplacingMergeTree staging table. - ---- - -## Exercise: Engine Selection for NYC Taxi Tables - -For each table below: -1. Determine the update pattern from the description -2. Identify the version column (if any) -3. Select the correct engine -4. Write a one-sentence reasoning - -Fill in the `?` cells: - -| Table | Update Pattern | Version Column | Recommended Engine | Reasoning | -|-------|----------------|----------------|--------------------|-----------| -| `fact_trips` | Trips can be corrected after the fact (fare adjustments, status changes) — same `trip_id` re-inserted with new values | ? | ? | ? | -| `agg_hourly_zone_trips` | dbt recalculates the last 2 hours on each run and re-inserts results | ? | ? | ? | -| `dim_taxi_zones` | dbt runs a full rebuild on each run (atomic table swap (full rebuild)) | — | ? | ? | -| `dim_payment_type` | dbt runs a full rebuild on each run (atomic table swap (full rebuild)) | — | ? | ? | -| `dim_vendor` | dbt runs a full rebuild on each run (atomic table swap (full rebuild)) | — | ? | ? | -| `mv_hourly_revenue` | ClickHouse REFRESHABLE MV recalculates on a schedule | — | ? | ? | - -**Hints:** -- `fact_trips`: What does "same trip_id re-inserted with new values" mean for deduplication? -- `agg_hourly_zone_trips`: What happens when dbt inserts a new row for `(2024-01-01 14:00, zone_id=1)` that already exists from the last run? -- `dim_*` tables: dbt's `table` materialization in dbt-clickhouse does atomic table swap (full rebuild), replacing all rows on every run. Does this require deduplication? -- `mv_hourly_revenue`: A REFRESHABLE MV replaces its entire result set on each refresh. What engine fits a table that is always fully replaced? - ---- - -## Reflection Questions - -Before checking the answer key, answer these questions in your own words: - -1. Why can't you use `MergeTree()` for `fact_trips` if trips can be updated? - -2. What happens if you query `fact_trips` without `FINAL` on a `ReplacingMergeTree` table? - -3. Why is `AggregatingMergeTree` not needed for `agg_hourly_zone_trips` even though it's an aggregation table? - ---- - -

-▶ Answer Key — try the exercises first before expanding - -| Table | Update Pattern | Version Column | Recommended Engine | Reasoning | -|-------|----------------|----------------|--------------------|-----------| -| `fact_trips` | Upsert (trip corrections) | `updated_at` | `ReplacingMergeTree(updated_at)` | Same trip_id can arrive twice with different fare/status values. RMT keeps the row with the highest `updated_at`. Always query with `FINAL`. | -| `agg_hourly_zone_trips` | Upsert (rolling 2-hr recalculation) | `updated_at` | `ReplacingMergeTree(updated_at)` | dbt recalculates the last 2 hours and re-inserts. Without RMT, old and new aggregates accumulate and double-count. `updated_at` set to `now()` on each dbt run ensures the latest values win. | -| `dim_taxi_zones` | Full reload | — | `MergeTree()` | dbt's `table` materialization does atomic table swap (full rebuild), so there is never more than one generation of rows. No deduplication needed. | -| `dim_payment_type` | Full reload | — | `MergeTree()` | Same as `dim_taxi_zones`. | -| `dim_vendor` | Full reload | — | `MergeTree()` | Same as `dim_taxi_zones`. | -| `mv_hourly_revenue` | Full replacement on each REFRESH | — | `MergeTree()` | REFRESHABLE MVs replace their entire result set atomically. No duplicates can accumulate. | - -**Reflection answers:** - -1. With `MergeTree`, every version of a trip row is kept permanently. If trip `T001` is inserted twice with different fare amounts, both rows appear in `SELECT * ... WHERE trip_id = 'T001'`, and `SUM(fare_amount)` doubles. ReplacingMergeTree deduplicates on merge so only the latest version is retained. - -2. Without `FINAL`, ClickHouse may return multiple versions of the same row — both the original insert and any updates — because background merges are asynchronous. Aggregates (SUM, COUNT) over the full table will over-count until the next merge runs. `FINAL` forces synchronous deduplication at read time. - -3. `AggregatingMergeTree` is for tables where rows store *partial aggregate states* (binary `AggregateFunction` columns) that need to be merged across rows. `agg_hourly_zone_trips` stores final values (INT, FLOAT) that must be deduplicated via upsert — the latest row wins, it is not combined with prior rows. `ReplacingMergeTree` is correct here; `AggregatingMergeTree` would require rewriting all queries to use `-Merge` suffix combiners. - -
- ---- - -## Transfer to migration-plan.md - -Once you have filled in this worksheet, copy your engine decisions to Section 3 of `migration-plan.md` and check off: - -``` -- [ ] Engine selection: completed -``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md deleted file mode 100644 index c8166cc..0000000 --- a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/02_sort_key_design.md +++ /dev/null @@ -1,157 +0,0 @@ -# Worksheet 2: Sort Key Design (ORDER BY) - -**Time estimate:** 20–25 minutes -**Reference:** [MergeTree Engines](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/mergetree-engines) — ORDER BY Design section - ---- - -## Concept - -ClickHouse's `ORDER BY` clause is not decorative. It defines the **primary index** — a sparse, block-level index that allows ClickHouse to skip irrelevant blocks of data when evaluating `WHERE` clauses. It also determines the physical sort order of data within parts, which affects compression. - -**The wrong ORDER BY = slow queries + wasted storage.** A UUID-first ORDER BY means no block skipping for any analytical query (UUIDs are random; there is no sortable prefix). A date-first ORDER BY means queries filtering on date skip most of the table. - -### Three rules for ORDER BY design - -**Rule 1: Derive from query filters, not from the source schema.** -Look at the `WHERE`, `GROUP BY`, and `JOIN` columns across your most frequent queries. The most-filtered columns should probably appear in `ORDER BY` (Provided it is not too high cardinality). The source table's primary key (if any) is usually irrelevant. - -**Rule 2: Low cardinality first, high cardinality last.** -ClickHouse's primary index has one entry per ~8192 rows (a *granule*). Low-cardinality columns (e.g., `toStartOfMonth(date)` = ~48 distinct values over 4 years) cluster many rows together — the index can skip entire granules. High-cardinality columns (e.g., `trip_id` = 50M distinct values) are unique per row — putting them first means the index cannot skip anything. - -**Rule 3: For ReplacingMergeTree, end with the unique row identifier.** -The deduplication key is the full `ORDER BY` tuple. If `trip_id` is missing from `ORDER BY`, two different trips with the same `pickup_at` and no further columns would be treated as duplicates. Put `trip_id` last to ensure uniqueness without hurting index performance. - ---- - -## Exercise: Query Workload Analysis - -Before designing sort keys, analyze what columns the queries actually filter on. - -The NYC Taxi lab has 7 representative queries. For each, identify the filter columns (WHERE, GROUP BY/ORDER BY on fact tables, JOIN keys): - -| Query | Description | Filter / Group Columns | Most selective filter | -|-------|-------------|----------------------|----------------------| -| Q1 | Hourly revenue by borough (DATE_TRUNC, DATEADD) | `pickup_at` (date range), `borough` (via dim join) | ? | -| Q2 | Rolling 7-day avg distance (window function) | `pickup_at::DATE` (GROUP BY — no WHERE filter) | ? | -| Q3 | Top 10 trips per zone (QUALIFY) | `pickup_at` (single day: yesterday), `pickup_borough` (PARTITION BY in window) | ? | -| Q4 | Driver ratings by zone (LATERAL FLATTEN / JSON) | JSON metadata fields (`driver` not null, `driver.rating` not null) | ? | -| Q5 | Surge pricing analysis (VARIANT colon-path) | `pickup_at` (date range) | ? | -| Q6 | Hourly aggregation (MERGE equivalent) | `hour_bucket`, `zone_id` | ? | -| Q7 | CDC stream lag measurement | `trip_id` (point lookup) | ? | - -Fill in the "Most selective filter" column: which column, when filtered, eliminates the most rows? - ---- - -## Exercise: Cardinality Estimation - -For each candidate ORDER BY column, estimate its cardinality over the 4-year, 50M-row dataset: - -| Column | Estimated Distinct Values | Cardinality Class | -|--------|--------------------------|------------------| -| `toStartOfMonth(pickup_at)` | ~48 (4 years × 12 months) | Very low | -| `toStartOfDay(pickup_at)` | ~1,460 (4 years × 365 days) | Low | -| `pickup_at` (DateTime64) | ~? | Medium-high | -| `pickup_location_id` | 265 (TLC zones) | Very low | -| `zone_id` | 265 | Very low | -| `hour_bucket` (by hour) | ~35,040 (4 years × 24 × 365) | Medium | -| `borough` | 6 | Extremely low | -| `payment_type` | 6 | Extremely low | -| `vendor_id` | 3 | Extremely low | -| `trip_id` (String UUID) | 50,000,000 | Extremely high | - -Fill in the `pickup_at` estimated cardinality. (Hint: the trip producer inserts ~60 trips/minute. What is 60 trips/minute × 4 years in seconds?) - ---- - -## Exercise: Sort Key Design - -Using your query workload analysis and cardinality estimates, propose an `ORDER BY` for each table: - -**Rules reminder:** -- Low cardinality first → most block-skipping -- Columns that appear in WHERE/GROUP BY of multiple queries → include them -- For ReplacingMergeTree tables → end with the unique row identifier -- Don't include columns that are never filtered on - -| Table | Engine | Proposed ORDER BY | Reasoning | -|-------|--------|-------------------|-----------| -| `trips_raw` | ReplacingMergeTree(_synced_at) | `(?, ?)` | *Python migration script loads data; time-range queries filter on pickup_at; trip_id must be in ORDER BY as the RMT dedup key* | -| `fact_trips` | ReplacingMergeTree(updated_at) | `(?, ?, ?)` | *Q1-Q7 all filter on pickup_at; trip_id needed for RMT uniqueness* | -| `agg_hourly_zone_trips` | ReplacingMergeTree(updated_at) | `(?, ?)` | *Aggregation queries filter on hour_bucket and zone_id* | -| `dim_taxi_zones` | MergeTree | `(location_id)` | *(filled as example — dim tables are small, lookup by PK)* | - -**Hints for `trips_raw`:** -- CDC lookups (`WHERE trip_id = ?`) need trip_id in ORDER BY for efficient point lookups -- Analytical range queries (`WHERE pickup_at BETWEEN ...`) benefit from pickup_at - -**Hints for `fact_trips`:** -- All 7 analytical queries filter on `pickup_at` in some form -- `toStartOfMonth(pickup_at)` as the first column groups an entire month's data into adjacent blocks — this enables coarse-grained block skipping even without PARTITION BY -- `trip_id` must be last (RMT dedup key) - -**Hints for `agg_hourly_zone_trips`:** -- Q6 groups by `hour_bucket` and `zone_id` — these are the only filter columns -- Which has lower cardinality? - ---- - -## Reflection Questions - -1. Why does `toStartOfMonth(pickup_at)` appear as the first column in `fact_trips` ORDER BY, rather than just `pickup_at`? - -2. If Q7 does a point lookup by `trip_id` on `trips_raw`, should `trip_id` be first in `trips_raw`'s ORDER BY? Why or why not? - -3. `dim_taxi_zones` has only 265 rows. Does the ORDER BY choice matter for this table? Explain. - ---- - -
-▶ Answer Key — try the exercises first before expanding - -### Query Workload Analysis - -| Query | Most selective filter | -|-------|-----------------------| -| Q1 | `pickup_at` (date range eliminates 95%+ of rows before borough join) | -| Q2 | No WHERE filter — full table scan grouped by `pickup_at::DATE` | -| Q3 | `pickup_at` (single day — eliminates ~99.7% of 4-year dataset) | -| Q4 | JSON metadata presence check — not a sortable/skippable column | -| Q5 | `pickup_at` (date range) | -| Q6 | `hour_bucket` (specific time bucket) | -| Q7 | `trip_id` (UUID point lookup — unique per row) | - -### Cardinality: pickup_at - -60 trips/min × 60 min/hr × 24 hr/day × 365 days/yr × 4 years = ~126M possible minute-slots across 4 years. At second precision, even more. The 50M trips span ~2.1M distinct seconds → very high cardinality. Using `pickup_at` raw as the first ORDER BY column gives poor block skipping. `toStartOfMonth(pickup_at)` gives ~48 distinct values — much more useful as a first sort key. - -### Sort Key Answers - -| Table | Proposed ORDER BY | Reasoning | -|-------|-------------------|-----------| -| `trips_raw` | `(pickup_at, trip_id)` | Time-range analytical queries benefit from pickup_at first. trip_id for CDC point lookups (RMT uses it as dedup key if needed). | -| `fact_trips` | `(toStartOfMonth(pickup_at), pickup_at, trip_id)` | Month prefix clusters entire months together (block-level skipping). pickup_at narrows within month. trip_id ensures RMT uniqueness. Q1-Q7 all filter on pickup_at in some form — no query filters on trip_id alone on the fact table. **Production alternative:** `PARTITION BY toStartOfMonth(pickup_at)` + `ORDER BY (pickup_at, trip_id)` is equally valid — partitioning handles month-level pruning at the filesystem level, freeing the ORDER BY to focus purely on within-partition range scans. The lab uses the function-in-ORDER BY approach to keep the DDL simple, but both are production-grade patterns. | -| `agg_hourly_zone_trips` | `(hour_bucket, zone_id)` | Both columns appear in Q6. hour_bucket has medium cardinality (~35K); zone_id has low cardinality (265). In query patterns, `WHERE hour_bucket >= X` narrows more than `WHERE zone_id = Y`, but zone_id is a secondary filter. Both should be in ORDER BY. | - -### Reflection Answers - -1. `toStartOfMonth(pickup_at)` as the first column groups all trips in a calendar month into adjacent storage blocks. ClickHouse's sparse index can skip all blocks outside the requested date range at month granularity, before `pickup_at` narrows further within the month. If `pickup_at` were first alone, every granule would cover a mix of timestamps with no coarse structure — the index would still work but less efficiently for month-level range scans. - - > **Production note:** An equally valid design is `PARTITION BY toStartOfMonth(pickup_at)` + `ORDER BY (pickup_at, trip_id)`. `PARTITION BY` handles month-level pruning at the filesystem level (entire partition directories are skipped before any index is consulted), while `ORDER BY` focuses purely on within-partition range scans. The trade-off: partitioning creates more files and can hurt performance if partitions are too small (e.g., daily partitions on a sparse dataset). For 50M rows across 4 years (~1M rows/month), monthly partitioning is a safe choice. The lab keeps `toStartOfMonth` in ORDER BY to avoid introducing `PARTITION BY` as a separate concept, but SAs should know both patterns. - -2. No. `trip_id` has extremely high cardinality (UUID = 50M distinct values). Putting it first means every block contains exactly one `trip_id` value range — the index is useless for anything except exact `trip_id` lookups. Since the vast majority of `trips_raw` queries are analytical (time-range scans), not CDC point lookups, `pickup_at` first is more valuable. The CDC lookup performance penalty for `trip_id` second is acceptable because CDC lookups are rare point-in-time operations, not the primary analytical workload. - -3. `dim_taxi_zones` has 265 rows — one granule (8192 rows per granule). ClickHouse will read the entire table in a single granule regardless of ORDER BY. The choice does not matter for performance. `(location_id)` is still good practice for clarity and for join performance when ClickHouse uses a hash join and needs to look up rows by location_id. - -
- ---- - -## Transfer to migration-plan.md - -Copy your ORDER BY decisions to Section 4 of `migration-plan.md` and check off: - -``` -- [ ] Sort key design: completed -``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md deleted file mode 100644 index e30edcd..0000000 --- a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/03_schema_translation.md +++ /dev/null @@ -1,204 +0,0 @@ -# Worksheet 3: Schema Translation - -**Time estimate:** 20–25 minutes -**Reference:** [Snowflake vs ClickHouse](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/snowflake-vs-clickhouse) — Section 2 (SQL Dialect Gaps) - ---- - -## Concept - -Type mapping and function translation are the most mechanical part of migration, but also the most error-prone if done carelessly. Snowflake and ClickHouse have different type systems with different semantics, and using the wrong type can cause silent precision loss, excessive storage, or broken query logic. - -**Key principles:** - -1. **Be explicit about precision.** Snowflake's `TIMESTAMP_NTZ` has nanosecond precision. ClickHouse's `DateTime` has only second precision — do not use it for version columns. Use `DateTime64(3, 'UTC')` for millisecond precision (matching most real-world requirements) or `DateTime64(9, 'UTC')` for nanoseconds. **This matters for correctness:** if a `ReplacingMergeTree` version column has only second precision, two updates arriving within the same second are non-deterministic — ClickHouse cannot determine which is newer. - -2. **Use the smallest correct integer type.** Snowflake's `INTEGER` is `NUMBER(38, 0)` — 38-digit fixed precision stored as a 128-bit value. ClickHouse has fixed-width integers: `Int8`, `Int16`, `Int32`, `Int64`, `UInt8`, `UInt16`, `UInt32`, `UInt64`. Choosing `UInt8` for `vendor_id` (values 1–3) saves 7 bytes per row vs `Int64`. At 50M rows, that is 350MB. - -3. **VARIANT → String.** ClickHouse has a native `JSON` type (available in v25.3+ as production-stable), but it is designed for truly dynamic schemas where the field names and structure are unknown at table creation time. For `trip_metadata` in this lab, the structure is known (`driver.rating`, `app.surge_multiplier`, etc.) — the better approach is to pre-flatten into typed columns during migration, or store as `String` and use `JSONExtract*` at query time. Use the `JSON` type when you genuinely cannot predict the schema: e.g., ingesting arbitrary customer event payloads where every event has different fields. - -4. **Float precision.** Snowflake's `FLOAT` maps to `Float64` in ClickHouse. For monetary amounts where exact decimal arithmetic is required, use `Decimal(18, 2)` — but for this lab, `Float64` is sufficient to match the source. - -5. **LowCardinality() — ClickHouse-only optimization.** Wrapping a type in `LowCardinality(String)` (or `LowCardinality(UInt8)`, etc.) tells ClickHouse to use a dictionary encoding for that column — values are stored as integer references to a dictionary rather than repeated strings. This typically gives 2–5x compression improvement and faster GROUP BY on string columns with fewer than ~10,000 distinct values. Snowflake has no equivalent; it handles this automatically. Good candidates in this lab: `pickup_borough` (6 values), `payment_type` (6 values), `vehicle_type`, `vendor_name`. - ---- - -## Exercise 1: Type Mapping for TRIPS_RAW - -Map each column from `NYC_TAXI_DB.RAW.TRIPS_RAW` to its ClickHouse type. Fill in the `?` cells: - -| Snowflake Column | Snowflake Type | ClickHouse Type | Notes | -|------------------|---------------|-----------------|-------| -| `TRIP_ID` | `VARCHAR(36)` | `String` | *(ClickHouse has a native [`UUID`](https://clickhouse.com/docs/sql-reference/data-types/uuid) type, but `String` is idiomatic when migrating from VARCHAR(36): it requires no casting, supports all string functions, and avoids UUID parsing overhead on insert)* | -| `PICKUP_DATETIME` | `TIMESTAMP_NTZ(9)` | ? | *Hint: what precision does DateTime64 support? What timezone?* | -| `DROPOFF_DATETIME` | `TIMESTAMP_NTZ(9)` | ? | *Same as PICKUP_DATETIME* | -| `PICKUP_LOCATION_ID` | `INTEGER` | ? | *Values 1–265. What is the smallest correct unsigned integer type?* | -| `DROPOFF_LOCATION_ID` | `INTEGER` | ? | *Same as PICKUP_LOCATION_ID* | -| `VENDOR_ID` | `INTEGER` | ? | *Values 1–3. What is the smallest correct unsigned integer type?* | -| `PASSENGER_COUNT` | `INTEGER` | ? | *Values 1–6 typically. What type?* | -| `TRIP_DISTANCE` | `FLOAT` | ? | *Decimal miles — precision matters somewhat but not critically* | -| `FARE_AMOUNT` | `FLOAT` | ? | *Monetary. Float64 is acceptable for this lab.* | -| `TIP_AMOUNT` | `FLOAT` | ? | *Same as FARE_AMOUNT* | -| `TOTAL_AMOUNT` | `FLOAT` | ? | *Same as FARE_AMOUNT* | -| `PAYMENT_TYPE_ID` | `INTEGER` | ? | *Values 1–6. What type?* | -| `TRIP_METADATA` | `VARIANT` | ? | *JSON blob — the key migration challenge* | -| `CREATED_AT` | `TIMESTAMP_NTZ(9)` | ? | *Row creation timestamp* | - ---- - -## Exercise 2: Type Mapping for FACT_TRIPS - -`FACT_TRIPS` adds computed/derived columns that were added by the dbt pipeline: - -| Snowflake Column | Snowflake Type | ClickHouse Type | Notes | -|------------------|---------------|-----------------|-------| -| `TRIP_ID` | `VARCHAR(36)` | `String` | *(filled)* | -| `PICKUP_AT` | `TIMESTAMP_NTZ(9)` | ? | | -| `DROPOFF_AT` | `TIMESTAMP_NTZ(9)` | ? | | -| `PICKUP_LOCATION_ID` | `INTEGER` | ? | | -| `ZONE_ID` | `INTEGER` | ? | *Foreign key to dim_taxi_zones; same cardinality as location IDs* | -| `FARE_AMOUNT` | `FLOAT` | ? | | -| `TRIP_DISTANCE` | `FLOAT` | ? | | -| `DRIVER_RATING` | `FLOAT` | ? | *Values 1.0–5.0; can be NULL (no rating)* | -| `PAYMENT_TYPE_ID` | `INTEGER` | ? | | -| `VENDOR_ID` | `INTEGER` | ? | | -| `UPDATED_AT` | `TIMESTAMP_NTZ(9)` | ? | *Version column for ReplacingMergeTree — precision matters* | - -**Note on nullable columns:** In ClickHouse, `Nullable(Float64)` has a slight performance overhead compared to `Float64`. For `driver_rating`, which is frequently NULL, you have two options: -- `Nullable(Float32)` — explicit null semantics; slower on aggregations due to null-tracking overhead (a separate bitmask column is stored alongside the data) -- `Float32` with `nan` or `-1.0` as sentinel value — faster but less conventional - -For the lab, use `Nullable(Float32)` for correctness. - ---- - -## Exercise 3: Function Translation - -Translate each Snowflake expression to its ClickHouse equivalent. These come directly from Q1–Q7 in `01-setup-snowflake/queries/`. - -| Snowflake Expression | Query | ClickHouse Equivalent | -|----------------------|-------|----------------------| -| `DATE_TRUNC('hour', pickup_at)` | Q1 | ? | -| `DATEADD('day', -7, CURRENT_DATE)` | Q1, Q3 | ? | -| `DATEDIFF('minute', pickup_at, dropoff_at)` | Q4 | ? | -| `TRIP_METADATA:driver.rating::FLOAT` | Q4, Q5 | ? | -| `TRIP_METADATA:app.surge_multiplier::FLOAT` | Q5 | ? | -| `QUALIFY ROW_NUMBER() OVER (PARTITION BY pickup_location_id ORDER BY fare_amount DESC) <= 10` | Q3 | ? — treated as a dialect gap in this lab; requires structural rewrite | -| `MERGE INTO fact_trips t USING staging s ON t.trip_id = s.trip_id WHEN MATCHED THEN UPDATE ...` | Q6 | ? — requires structural rewrite *(Hint: think about the engine and dbt strategy you chose in Worksheets 1 & 2)* | -| `SELECT METADATA$ACTION, METADATA$ISUPDATE FROM trips_cdc_stream` | Q7 | ? — no equivalent needed; live writes go directly to ClickHouse post-cutover; what pattern gives you current state? | - -**Hints:** -- `DATE_TRUNC('hour', ...)` → look at `toStartOfHour` family in ClickHouse docs -- `DATEADD('day', -7, ...)` → ClickHouse uses `today() - 7` for dates, or `now() - INTERVAL 7 DAY` -- `DATEDIFF('minute', t1, t2)` → ClickHouse has `dateDiff` (lowercase d) -- `TRIP_METADATA:driver.rating::FLOAT` → nested path requires two-level `JSONExtractFloat` -- QUALIFY → for this lab, treated as a dialect gap; wrap in a subquery: keep the window function, add `WHERE` outside (this rewrite is portable across all SQL engines) - ---- - -## Exercise 4: Non-Obvious Translation Decisions - -For each decision below, write your reasoning (not just the answer): - -**Decision 1:** `TRIP_METADATA` is `VARIANT` in Snowflake. You have three options in ClickHouse: -- `String` — store raw JSON; use `JSONExtract*` at query time -- `Map(String, String)` — structured key-value; loses nested structures -- `Tuple(...)` — fixed schema; breaks if JSON structure varies - -Which do you choose and why? - -*Your answer:* - ---- - -**Decision 2:** `FARE_AMOUNT` is `FLOAT` in Snowflake. In ClickHouse, you could use `Float64`, `Float32`, or `Decimal(18, 2)`. The lab has 50M rows. - -Which do you choose and why? (Consider: precision needs, storage cost, and whether fare amounts in this workload require exact decimal arithmetic.) - -*Your answer:* - ---- - -**Decision 3:** `PICKUP_LOCATION_ID` is `INTEGER` in Snowflake (values 1–265). In ClickHouse, you could use `Int32`, `Int16`, or `UInt16`. - -Calculate the storage savings of using `UInt16` instead of `Int32` at 50M rows: - -*Your calculation:* - ---- - -
-▶ Answer Key — try the exercises first before expanding - -### Exercise 1: TRIPS_RAW Types - -| Column | ClickHouse Type | Reasoning | -|--------|----------------|-----------| -| `PICKUP_DATETIME` | `DateTime64(3, 'UTC')` | Millisecond precision is sufficient; `'UTC'` makes timezone explicit. Nanosecond (`9`) is overkill for trip timestamps. | -| `DROPOFF_DATETIME` | `DateTime64(3, 'UTC')` | Same. | -| `PICKUP_LOCATION_ID` | `UInt16` | Values 1–265. `UInt8` max is 255 — too small. `UInt16` max is 65535 — correct and half the size of `Int32`. | -| `DROPOFF_LOCATION_ID` | `UInt16` | Same. | -| `VENDOR_ID` | `UInt8` | Values 1–3. `UInt8` max is 255 — correct. 1 byte per row vs 4 for Int32. | -| `PASSENGER_COUNT` | `UInt8` | Values 1–6. `UInt8` correct. | -| `TRIP_DISTANCE` | `Float64` | Float distance; no exact decimal requirement. Matches Snowflake FLOAT. | -| `FARE_AMOUNT` | `Float64` | See Exercise 4 Decision 2. | -| `TIP_AMOUNT` | `Float64` | Same. | -| `TOTAL_AMOUNT` | `Float64` | Same. | -| `PAYMENT_TYPE_ID` | `UInt8` | Values 1–6. | -| `TRIP_METADATA` | `String` | See Exercise 4 Decision 1. | -| `CREATED_AT` | `DateTime64(3, 'UTC')` | Row creation timestamp — milliseconds sufficient. | - -### Exercise 2: FACT_TRIPS Types - -| Column | ClickHouse Type | -|--------|----------------| -| `PICKUP_AT` | `DateTime64(3, 'UTC')` | -| `DROPOFF_AT` | `DateTime64(3, 'UTC')` | -| `PICKUP_LOCATION_ID` | `UInt16` | -| `ZONE_ID` | `UInt16` | -| `FARE_AMOUNT` | `Float64` | -| `TRIP_DISTANCE` | `Float64` | -| `DRIVER_RATING` | `Nullable(Float32)` | -| `PAYMENT_TYPE_ID` | `UInt8` | -| `VENDOR_ID` | `UInt8` | -| `UPDATED_AT` | `DateTime64(3, 'UTC')` — must be DateTime64 not DateTime; second precision could cause two updates in the same second to be non-deterministic | - -### Exercise 3: Function Translations - -| Snowflake Expression | ClickHouse Equivalent | -|----------------------|-----------------------| -| `DATE_TRUNC('hour', pickup_at)` | `toStartOfHour(pickup_at)` | -| `DATEADD('day', -7, CURRENT_DATE)` | `today() - 7` or `addDays(today(), -7)` | -| `DATEDIFF('minute', pickup_at, dropoff_at)` | `dateDiff('minute', pickup_at, dropoff_at)` | -| `TRIP_METADATA:driver.rating::FLOAT` | `JSONExtractFloat(trip_metadata, 'driver', 'rating')` | -| `TRIP_METADATA:app.surge_multiplier::FLOAT` | `JSONExtractFloat(trip_metadata, 'app', 'surge_multiplier')` | -| `QUALIFY ROW_NUMBER() OVER (...) <= 10` | `SELECT ... FROM (SELECT ..., ROW_NUMBER() OVER (...) AS rn FROM ...) WHERE rn <= 10` | -| `MERGE INTO ... WHEN MATCHED THEN UPDATE` | dbt `delete_insert` incremental strategy — no direct SQL equivalent | -| `SELECT METADATA$ACTION FROM stream` | No equivalent needed — live writes go directly to ClickHouse post-cutover; query the target table with `FINAL` for current deduplicated state | - -### Exercise 4: Decisions - -**Decision 1 — TRIP_METADATA → String** -Choose `String`. `Map(String, String)` loses nested structures (the rating is nested under `driver.rating`, not a flat key). `Tuple(...)` requires a fixed schema — if the JSON structure varies across trips, the Tuple definition fails or requires frequent schema migrations. `String` preserves the raw JSON exactly, and `JSONExtract*` handles any path at query time. The trade-off is that extraction is slower than native column access, but for VARIANT columns that are queried infrequently (only in Q4, Q5), this is acceptable. - -**Decision 2 — FARE_AMOUNT → Float64** -Use `Float64`. `Float32` has ~7 significant digits of precision — a $10,000.00 fare would be stored as $9,999.999... (rounding at 7 digits), which is wrong for financial reporting. `Decimal(18, 2)` gives exact precision but is slower on aggregations. For a lab environment where we're demonstrating analytics performance, `Float64` (15–16 significant digits) is the pragmatic choice. In a production financial system, `Decimal(18, 2)` would be correct. - -**Decision 3 — UInt16 vs Int32 storage savings** -- `Int32`: 4 bytes per row × 50,000,000 rows = 200,000,000 bytes = ~190 MB -- `UInt16`: 2 bytes per row × 50,000,000 rows = 100,000,000 bytes = ~95 MB -- Savings: ~95 MB per column before compression -- ClickHouse compression (LZ4 default) compresses sorted integer columns at ~5-10x for low-cardinality values → actual storage might be 10-20 MB per column regardless -- But: two `UInt16` columns (`pickup_location_id` + `dropoff_location_id`) on 50M rows saves ~190 MB raw storage and improves compression since small integers compress better - -
- ---- - -## Transfer to migration-plan.md - -Copy your type decisions and any non-obvious translation notes to Section 5 of `migration-plan.md` and check off: - -``` -- [ ] Schema translation: completed -``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md deleted file mode 100644 index b67083d..0000000 --- a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/04_migration_wave_plan.md +++ /dev/null @@ -1,183 +0,0 @@ -# Worksheet 4: Migration Wave Plan - -**Time estimate:** 15–20 minutes -**Reference:** [Module 03 — Provision and migrate](https://workshop.demohouse.cloud/docs/snowflake-migration/learner/03-provision-and-migrate) → [Module 04 — Rebuild the dbt pipeline](https://workshop.demohouse.cloud/docs/snowflake-migration/learner/04-rebuild-dbt) → [Module 05 — Benchmark and cutover](https://workshop.demohouse.cloud/docs/snowflake-migration/learner/05-benchmark-and-cutover) for execution order - ---- - -## Concept - -Database objects have dependencies. A view that queries `fact_trips` cannot be created before `fact_trips` exists. A materialized view that reads from `trips_raw` cannot be populated before `trips_raw` has data. Migrating in the wrong order causes create-table failures, empty results, or incomplete data that is hard to diagnose. - -**The solution: wave planning.** Organize every object into a numbered wave where each wave contains only objects whose dependencies are satisfied by prior waves. - -**Complexity grading** helps prioritize migration effort: -- **Grade A** — trivial: standard table, no special logic, straightforward type mapping -- **Grade B** — medium: dbt incremental model with ClickHouse-specific engine; requires testing -- **Grade C** — complex: requires SQL rewrite (QUALIFY, MERGE INTO, VARIANT); test carefully -- **Grade D** — requires redesign: Snowflake-specific feature with no direct equivalent (Streams → producer cutover to ClickHouse, Tasks → Refreshable MVs or scheduled dbt runs) - ---- - -## Exercise 1: Dependency DAG - -Map the dependencies between all objects in the NYC Taxi workload. Draw a line from each object to the objects it depends on. - -``` -Objects to map: - Source (Snowflake): - - TRIPS_RAW (raw table) - - TRIPS_CDC_STREAM (stream on TRIPS_RAW) - - CDC_CONSUME_TASK (task reading TRIPS_CDC_STREAM) - - HOURLY_AGG_TASK (task writing AGG_HOURLY_ZONE_TRIPS) - - ClickHouse targets (via dbt or Python migration script): - - trips_raw (ClickHouse base table) - - stg_trips (dbt view) - - stg_taxi_zones (dbt view) - - int_trips_enriched (dbt ephemeral CTE) - - fact_trips (dbt incremental) - - agg_hourly_zone_trips (dbt incremental) - - dim_taxi_zones (dbt table) - - dim_payment_type (dbt table) - - dim_vendor (dbt table) - - taxi_zones_dict (ClickHouse dictionary) - - mv_hourly_revenue (ClickHouse Refreshable MV) -``` - -Fill in the dependency table: - -| Object | Depends On | Notes | -|--------|------------|-------| -| `trips_raw` (ClickHouse) | *(nothing — first object created)* | Schema created by dbt before the Python migration script loads data | -| `stg_trips` | ? | | -| `stg_taxi_zones` | ? | | -| `int_trips_enriched` | ? | | -| `fact_trips` | ? | | -| `agg_hourly_zone_trips` | ? | | -| `dim_taxi_zones` | ? | | -| `dim_payment_type` | ? | *Source data is static reference data — does it depend on trips_raw?* | -| `dim_vendor` | ? | | -| `taxi_zones_dict` | ? | *Dictionary reads from dim_taxi_zones at creation time* | -| `mv_hourly_revenue` | ? | *Refreshable MV queries fact_trips on each refresh* | - ---- - -## Exercise 2: Migration Wave Assignment - -Using the dependency map above, assign each object to a migration wave. Objects in the same wave can be created/loaded in parallel. Objects in later waves must wait for earlier waves to complete. - -Fill in the wave assignment: - -| Wave | Objects | What happens | -|------|---------|--------------| -| Wave 0 | `trips_raw` (schema only), `dim_*` tables | ? | -| Wave 1 | Python bulk load (`scripts/02_migrate_trips.py`), `stg_trips`, `stg_taxi_zones` | ? | -| Wave 2 | ? | ? | -| Wave 3 | ? | ? | -| Wave 4 | ? | ? | - -**Constraint:** The Python bulk load (`scripts/02_migrate_trips.py`) must complete before Wave 2 dbt models run — `stg_trips`, `int_trips_enriched`, and `fact_trips` all read from `trips_raw`, which must have data. - ---- - -## Exercise 3: Complexity Grading - -Grade each object A/B/C/D and explain the primary migration challenge: - -| Object | Grade | Primary Challenge | -|--------|-------|------------------| -| `trips_raw` | ? | | -| `stg_trips` | ? | *Contains JSONExtract for TRIP_METADATA — straightforward but requires JSON path knowledge* | -| `stg_taxi_zones` | ? | | -| `int_trips_enriched` | ? | | -| `fact_trips` | ? | *ReplacingMergeTree, delete_insert incremental, QUALIFY rewrite in Q3* | -| `agg_hourly_zone_trips` | ? | | -| `dim_taxi_zones` | ? | | -| `dim_payment_type` | ? | | -| `dim_vendor` | ? | | -| `taxi_zones_dict` | ? | *ClickHouse-specific syntax; no Snowflake equivalent* | -| `mv_hourly_revenue` | ? | *Refreshable MV syntax is ClickHouse-specific; REFRESH EVERY syntax* | -| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | ? | *Snowflake-specific; replaced by direct producer cutover to ClickHouse in Part 3* | - ---- - -## Exercise 4: Risk Register - -For each Grade C/D object, identify: -1. What can go wrong -2. How you will verify correctness after migration - -| Object | Risk | Verification | -|--------|------|--------------| -| `fact_trips` | ? | | -| `agg_hourly_zone_trips` | ? | | -| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | ? | | - ---- - -
-▶ Answer Key — try the exercises first before expanding - -### Exercise 1: Dependency Table - -| Object | Depends On | -|--------|------------| -| `trips_raw` | Nothing (created first; schema before data) | -| `stg_trips` | `trips_raw` (reads from base table) | -| `stg_taxi_zones` | raw source `taxi_zones` — seeded by `scripts/00_seed_zones.sql` into `default.taxi_zones` (distinct from the analytics `dim_taxi_zones` model) | -| `int_trips_enriched` | `stg_trips`, `stg_taxi_zones` (joins both staging views) | -| `fact_trips` | `int_trips_enriched` (dbt CTE chain) | -| `agg_hourly_zone_trips` | `int_trips_enriched` (dbt CTE chain) | -| `dim_taxi_zones` | Reference data (static CSV seed — no CH dependency) | -| `dim_payment_type` | Reference data (static — no dependency) | -| `dim_vendor` | Reference data (static — no dependency) | -| `taxi_zones_dict` | `dim_taxi_zones` (reads from it at dictionary creation time) | -| `mv_hourly_revenue` | `fact_trips` (refreshable MV queries fact_trips) | - -### Exercise 2: Migration Wave Assignment - -| Wave | Objects | What happens | -|------|---------|--------------| -| Wave 0 | `trips_raw` schema, `dim_taxi_zones`, `dim_payment_type`, `dim_vendor` | dbt creates empty tables; reference dims populated immediately (static data) | -| Wave 1 | Python bulk load (`scripts/02_migrate_trips.py`) → `trips_raw` | 50M rows loaded from Snowflake; verify row count parity with `scripts/01_verify_migration.sh` | -| Wave 2 | `stg_trips`, `stg_taxi_zones`, `int_trips_enriched` (ephemeral), `fact_trips`, `agg_hourly_zone_trips` | `dbt run` builds the analytics layer from loaded data | -| Wave 3 | `taxi_zones_dict`, `mv_hourly_revenue` | Dictionary created after `dim_taxi_zones` is populated; refreshable MV created after `fact_trips` has data | -| Wave 4 | Producer cutover (`scripts/03_cutover.sh`) | Stop Snowflake producer; run final migration delta; start ClickHouse producer writing directly to ClickHouse Cloud | - -### Exercise 3: Complexity Grading - -| Object | Grade | Challenge | -|--------|-------|-----------| -| `trips_raw` | A | ReplacingMergeTree(_synced_at); migration retries and post-cutover producer retries require idempotent inserts — same decision as Worksheet 1 | -| `stg_trips` | B | JSONExtract for TRIP_METADATA; requires JSON path knowledge and testing | -| `stg_taxi_zones` | A | Passthrough view; trivial translation | -| `int_trips_enriched` | A | Ephemeral CTE; SQL differences are handled in parent models | -| `fact_trips` | C | ReplacingMergeTree engine; `delete_insert` incremental strategy; QUALIFY treated as a dialect gap in Q3 (subquery rewrite); must use FINAL in queries | -| `agg_hourly_zone_trips` | B | ReplacingMergeTree; rolling window recalculation logic; `delete_insert` strategy with 2-hr window | -| `dim_taxi_zones` | A | Static reference; full reload; trivial | -| `dim_payment_type` | A | Static reference; trivial | -| `dim_vendor` | A | Static reference; trivial | -| `taxi_zones_dict` | B | ClickHouse-specific `CREATE DICTIONARY` syntax; `HASHED()` layout; dictGet() at query time | -| `mv_hourly_revenue` | B | `REFRESH EVERY 180 SECOND` syntax; must verify it replaces data atomically | -| `TRIPS_CDC_STREAM` / `CDC_CONSUME_TASK` | D | Snowflake Streams and Tasks have no direct ClickHouse equivalent; replaced by direct producer cutover in Part 3 — the live trip producer writes to ClickHouse Cloud after cutover | - -### Exercise 4: Risk Register - -| Object | Risk | Verification | -|--------|------|--------------| -| `fact_trips` | Queries without FINAL over-count rows during merge lag period; `delete_insert` partition range must match ORDER BY prefix to avoid deleting non-target rows | `SELECT COUNT(*) FROM fact_trips FINAL` matches Snowflake `COUNT(*)` within CDC lag tolerance; run `dbt test` and compare Q3 output between systems | -| `agg_hourly_zone_trips` | Rolling 2-hr recalculation window in dbt must correctly scope the delete range to avoid deleting hours outside the recalc window | Spot-check specific `(hour_bucket, zone_id)` pairs against Snowflake; verify no older hours disappear after a dbt run | -| `TRIPS_CDC_STREAM` / producer cutover | Migration script interrupted mid-run leaves a row-count gap; producer retry after cutover may re-insert trips already in ClickHouse | Run `scripts/01_verify_migration.sh` for row-count parity; `ReplacingMergeTree(_synced_at)` makes producer retries idempotent | - -
- ---- - -## Transfer to migration-plan.md - -Copy your wave plan and complexity grades to Section 6 of `migration-plan.md` and check off: - -``` -- [ ] Migration wave plan: completed -``` diff --git a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md b/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md deleted file mode 100644 index 420263e..0000000 --- a/workshop_public/snowflake_migration_lab/02-plan-and-design/worksheets/05_dbt_model_design.md +++ /dev/null @@ -1,191 +0,0 @@ -# Worksheet 5: dbt Model Design - -**Time estimate:** 15–20 minutes -**Reference:** [dbt on ClickHouse](https://workshop.demohouse.cloud/docs/snowflake-migration/reference/dbt-on-clickhouse) - ---- - -## Concept - -Worksheets 1–4 produced the *what*: which engine, which ORDER BY, which ClickHouse types. This worksheet produces the *how*: how those decisions are expressed as dbt configuration before writing any SQL. - -A dbt-clickhouse model has three layers of configuration that Snowflake developers don't need to think about: - -1. **Materialization** — which physical object does dbt create (view, table, incremental, ephemeral)? -2. **Engine config** — for tables and incremental models, which ClickHouse engine and version column? -3. **Incremental strategy** — for incremental models, how does dbt handle new/updated rows? - -There is also a correctness concern unique to ReplacingMergeTree: - -4. **FINAL placement** — where in the dbt DAG does `FINAL` go to guarantee deduplicated reads? - ---- - -## Exercise 1: Materialization Selection - -For each model below, choose the correct dbt materialization type (`view`, `ephemeral`, `table`, or `incremental`) and write a one-sentence reasoning. - -Use the update pattern description to guide your choice: -- Insert-only read from a source? → `view` -- Pure join logic, not queried directly? → `ephemeral` -- Full rebuild on every dbt run, no partial updates? → `table` -- Only new/changed rows processed per run? → `incremental` - -Fill in the `?` cells: - -| Model | Layer | Update Pattern | Materialization | Why | -|-------|-------|----------------|-----------------|-----| -| `stg_trips` | staging | Read + clean from `trips_raw`; no writes to this model | ? | ? | -| `stg_taxi_zones` | staging | Read + clean from raw `taxi_zones` source (seeded by `scripts/00_seed_zones.sql`); no writes | ? | ? | -| `int_trips_enriched` | intermediate | Joins `stg_trips` with zone, payment, vendor, and date dims | ? | ? | -| `fact_trips` | analytics | One row per trip; trips can be corrected (fare adjustments re-insert same `trip_id`) | ? | ? | -| `agg_hourly_zone_trips` | analytics | dbt recalculates the last 2 hours on every run and re-inserts results | ? | ? | -| `dim_taxi_zones` | analytics | 265 static zones; dbt rebuilds the entire table from scratch on every run | ? | ? | -| `dim_payment_type` | analytics | 6 static types; dbt rebuilds from scratch on every run | ? | ? | -| `dim_vendor` | analytics | 3 vendors; dbt rebuilds from scratch on every run | ? | ? | - ---- - -## Exercise 2: Engine Configuration - -Only models with `table` or `incremental` materialization need a ClickHouse engine. Views and ephemeral models have no engine. - -For each model below, fill in the `ENGINE` and `Version Column` columns. Reference your answers from Worksheet 1 (engine selection) — the dbt engine config should match the design decisions you already made. - -| Model | Materialization | ENGINE | Version Column | Why this engine? | -|-------|----------------|--------|----------------|-----------------| -| `fact_trips` | incremental | ? | ? | ? | -| `agg_hourly_zone_trips` | incremental | ? | ? | ? | -| `dim_taxi_zones` | table | ? | — | ? | -| `dim_payment_type` | table | ? | — | ? | -| `dim_vendor` | table | ? | — | ? | - -**Hint:** Cross-check your answers against your completed Section 3 in `migration-plan.md`. The dbt engine config is the implementation of the engine decisions you made in Worksheet 1. - ---- - -## Exercise 3: Incremental Strategy Design - -For each incremental model, design the full incremental configuration: `unique_key`, `incremental_strategy`, and the SQL filter that goes inside the `is_incremental()` guard block. - -```sql --- Template for the is_incremental() block: -{% if is_incremental() %} - WHERE -{% endif %} -``` - -Fill in the `?` cells: - -| Model | `unique_key` | `incremental_strategy` | Incremental filter (inside `is_incremental()`) | Why this filter? | -|-------|-------------|----------------------|------------------------------------------------|-----------------| -| `fact_trips` | ? | ? | `WHERE updated_at > ?` | ? | -| `agg_hourly_zone_trips` | ? | ? | `WHERE pickup_at >= ?` | ? | - -**Hints:** -- `fact_trips` uses a high-watermark pattern — it processes rows newer than the latest row already in the target. Which column should the watermark be on: `pickup_at` or `updated_at`? Think about what happens when a fare adjustment re-inserts the same `trip_id`. -- `agg_hourly_zone_trips` produces hourly aggregates. If you used a high-watermark like `fact_trips`, what would happen to the boundary hour that was partially calculated in the previous run? -- For `agg_hourly_zone_trips`, why is the composite key `[hour_bucket, zone_id]` correct here, while `fact_trips` uses only `trip_id`? - ---- - -## Exercise 4: FINAL Placement - -ReplacingMergeTree deduplication is asynchronous. `FINAL` forces it at read time — but placing FINAL in the wrong model has correctness or performance consequences. - -For each model below, decide whether `FINAL` should appear in the model's `FROM` clause (i.e., `FROM source FINAL` or `FROM {{ ref(...) }} FINAL`), and explain why: - -| Model | Reads from | FINAL in this model's FROM clause? | Why? | -|-------|-----------|-------------------------------------|------| -| `stg_trips` | `trips_raw` (ReplacingMergeTree) | ? | ? | -| `int_trips_enriched` | `stg_trips` (view) | ? | ? | -| `fact_trips` | `int_trips_enriched` (ephemeral) | ? | ? | - -**Hints:** -- `trips_raw` is ReplacingMergeTree — it can have duplicate `trip_id` rows from migration script retries or post-cutover producer retries. Where should this be resolved? -- `stg_trips` is a view, not an RMT table. Does reading from a view require FINAL? -- `fact_trips` uses `delete_insert`. Does that mean the table is already clean without FINAL? - ---- - -## Reflection Questions - -Answer these before checking the answer key: - -1. `int_trips_enriched` could be a `view` materialization (it reads from staging views). Why is `ephemeral` a better choice here? - -2. `agg_hourly_zone_trips` recalculates using `now() - INTERVAL 2 HOUR`. What would happen if you used `max(pickup_at)` instead (the same pattern as `fact_trips`)? - -3. You have `delete_insert` on `fact_trips` and FINAL in `stg_trips`. What would happen if you removed FINAL from `stg_trips` but kept delete_insert on `fact_trips`? - ---- - -
-▶ Answer Key — try the exercises first before expanding - -### Exercise 1: Materialization Selection - -| Model | Materialization | Why | -|-------|-----------------|-----| -| `stg_trips` | `view` | Reads and cleans source data; no updates to this model; views have zero storage cost and are always fresh | -| `stg_taxi_zones` | `view` | Same as stg_trips — passthrough cleaning of a source table | -| `int_trips_enriched` | `ephemeral` | Pure join logic consumed only by `fact_trips`; no need for a physical table; inlined as a CTE avoids an unnecessary database object | -| `fact_trips` | `incremental` | Trips can be corrected after the fact; only new and updated rows should be processed per run | -| `agg_hourly_zone_trips` | `incremental` | Rolling 2-hour recalculation is an incremental pattern — process recent rows, not all 50M | -| `dim_taxi_zones` | `table` | 265 static zones; full rebuild on every dbt run; no partial updates needed | -| `dim_payment_type` | `table` | 6 static types; same reasoning as dim_taxi_zones | -| `dim_vendor` | `table` | 3 vendors; same reasoning | - -### Exercise 2: Engine Configuration - -| Model | ENGINE | Version Column | Why | -|-------|--------|----------------|-----| -| `fact_trips` | `ReplacingMergeTree(updated_at)` | `updated_at` | Trips can be corrected; `updated_at` set to `now()` on each insert means the latest correction wins during RMT dedup | -| `agg_hourly_zone_trips` | `ReplacingMergeTree(updated_at)` | `updated_at` | Rolling recalculation re-inserts aggregates for (hour_bucket, zone_id) pairs; RMT ensures stale aggregates are removed on background merge | -| `dim_taxi_zones` | `MergeTree()` | — | Full reload by dbt means atomic table swap (full rebuild) on every run; no duplicate rows can accumulate; no dedup needed | -| `dim_payment_type` | `MergeTree()` | — | Same as dim_taxi_zones | -| `dim_vendor` | `MergeTree()` | — | Same as dim_taxi_zones | - -### Exercise 3: Incremental Strategy Design - -| Model | `unique_key` | `incremental_strategy` | Incremental filter | Why | -|-------|-------------|----------------------|-------------------|-----| -| `fact_trips` | `trip_id` | `delete_insert` | `WHERE updated_at > (SELECT max(updated_at) FROM {{ this }})` | High-watermark on `updated_at` processes both new trips and corrected trips (fare adjustments re-insert the same `trip_id` with the same `pickup_at` but a newer `updated_at`). A `pickup_at` watermark would silently miss corrections — the pickup time never changes. | -| `agg_hourly_zone_trips` | `[hour_bucket, zone_id]` | `delete_insert` | `WHERE pickup_at >= now() - INTERVAL 2 HOUR` | Rolling window re-aggregates the last 2 hours on every run; this ensures the current boundary hour is fully recalculated even if only partial data was available at the previous run's cutoff | - -**Why `[hour_bucket, zone_id]` for agg, not just `trip_id`?** -`agg_hourly_zone_trips` has one row per `(hour_bucket, zone_id)` pair, not one row per trip. The composite key `[hour_bucket, zone_id]` is the natural unique key. `delete_insert` deletes all (hour_bucket, zone_id) pairs that appear in the new batch, then inserts the freshly computed aggregates. - -**Why `now() - INTERVAL 2 HOUR` breaks with `max(pickup_at)`?** -If `agg_hourly_zone_trips` used `max(pickup_at)` as its filter, it would process only trips with `pickup_at > (last run's max)`. Trips from the current incomplete hour would be counted in one run, then never re-aggregated — the partial hour count would be permanently wrong. The rolling 2-hour window forces re-computation of recent hours so that boundary hours are always complete. - -### Exercise 4: FINAL Placement - -| Model | FINAL in FROM clause? | Why | -|-------|----------------------|-----| -| `stg_trips` | **Yes** — `FROM trips_raw FINAL` | `trips_raw` is ReplacingMergeTree; it can have duplicate `trip_id` rows from the bulk-load + CDC overlap window. `stg_trips` is the single enforcement point: deduplicate here so every downstream model sees clean data | -| `int_trips_enriched` | **No** | `int_trips_enriched` reads from `stg_trips`, which is a view (not an RMT table). FINAL is irrelevant for views. | -| `fact_trips` | **No** (in the model itself) | `delete_insert` keeps `fact_trips` clean — there should be no duplicates after a completed run. Dashboards and dbt tests that query `fact_trips` directly use `FINAL` externally as a safety net. Adding FINAL inside the model's SELECT would also apply it to the `is_incremental()` subquery that reads `max(pickup_at)` from `{{ this }}` — unnecessary overhead on every incremental run. | - -### Reflection Answers - -1. **Why ephemeral over view for `int_trips_enriched`?** -A view would create a database object that materializes the JOIN on every downstream query. Since `int_trips_enriched` is only used by `fact_trips`, an ephemeral model inlines the JOIN as a CTE directly inside `fact_trips`'s query plan — one execution, no extra round-trips, no physical object to maintain. - -2. **What breaks with `max(pickup_at)` for `agg_hourly_zone_trips`?** -With a high-watermark filter, only trips with `pickup_at > last_run_max` would be processed. Trips arriving in the current incomplete hour were partially counted in the previous run (up to `last_run_max`), then frozen. When new trips arrive for that same hour, they would not be re-aggregated — the boundary hour's count would be permanently understated. The rolling 2-hour window forces re-computation of recent hours so partial hours are always corrected. - -3. **What happens if you remove FINAL from `stg_trips`?** -Without FINAL, `stg_trips` may return two rows for the same `trip_id` if `trips_raw` has an unmerged duplicate from the bulk + CDC overlap. `int_trips_enriched` would join those two rows, producing two enriched rows for the same trip. `delete_insert` on `fact_trips` would then delete and re-insert both — so `fact_trips` would also have two rows for that `trip_id`. They would eventually be deduplicated by RMT's background merge, but until then, SUM(fare_amount) would double-count corrected trips and `dbt test` with `unique` on `trip_id` would fail. - -
- ---- - -## Transfer to migration-plan.md - -Once you have completed this worksheet, fill in Section 10 of `migration-plan.md` with your answers and check off: - -``` -- [ ] dbt model design: completed -``` diff --git a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh index 6d59cb6..2f146ea 100755 --- a/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh +++ b/workshop_public/snowflake_migration_lab/03-migrate-to-clickhouse/setup.sh @@ -7,7 +7,7 @@ # # Everything else (dbt, migration script, verification, Superset, benchmark) # is done manually following the steps in the playbook at -# https://workshop.demohouse.cloud/docs/snowflake-migration — because those +# https://labs.demohouse.cloud/docs/snowflake-migration — because those # are the migration decisions you should make deliberately, not skip. # # Prerequisites: @@ -164,7 +164,7 @@ echo " Source it in any terminal: source .clickhouse_state" echo "" echo -e "${BOLD} ─── Next steps ───────────────────────────────────────────${RESET}" echo " Follow the playbook to complete the remaining steps:" -echo " https://workshop.demohouse.cloud/docs/snowflake-migration" +echo " https://labs.demohouse.cloud/docs/snowflake-migration" echo "" echo " Module 03 — Provision and migrate" echo " Step 1 ✓ Provision cluster (done)" diff --git a/workshop_public/snowflake_migration_lab/README.md b/workshop_public/snowflake_migration_lab/README.md index 1b59897..12c640b 100644 --- a/workshop_public/snowflake_migration_lab/README.md +++ b/workshop_public/snowflake_migration_lab/README.md @@ -5,7 +5,7 @@ you run during the ClickHouse Snowflake Migration Lab. It is not the instruction ## Start here -The playbook lives at **https://workshop.demohouse.cloud/docs/snowflake-migration**. +The playbook lives at **https://labs.demohouse.cloud/docs/snowflake-migration**. Begin with module 00, which covers the toolchain, the two cloud trial accounts, and the virtualenvs. Each module tells you which files here to run and in what order. Working @@ -17,12 +17,14 @@ several steps depend on state an earlier module created. | Directory | What it is | |---|---| | `01-setup-snowflake/` | Terraform, SQL, dbt project, trip producer, and Superset stack for the Snowflake source environment | -| `02-plan-and-design/` | Profiling scripts, five worksheets you fill in, and `migration-plan.md` | +| `02-plan-and-design/` | Profiling scripts and `migration-plan.md` (the five worksheets are interactive pages in the playbook) | | `03-migrate-to-clickhouse/` | ClickHouse Cloud Terraform, the migration script, the ClickHouse dbt project, benchmark and cutover scripts | | `04-evaluation/` | `assessment.md`, the blank assessment template | -The worksheets, `migration-plan.md`, and `assessment.md` are yours to edit. Everything -else you run as-is. +`migration-plan.md` and `assessment.md` are yours to edit. Everything else you run as-is. +The five design worksheets are no longer files here — they are interactive pages in the +playbook that check each answer as you go, and your answers live in your browser. Module +02 links all five. ## Credentials