From 97e76b691b69e83fb8df385f84837262c16d660f Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Mon, 18 May 2026 19:34:21 +0200 Subject: [PATCH 1/6] Split agent goal from human instructions (goal.md / content.md) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent prompts now receive only goal.md (outcome description) while human pages keep the full step-by-step content. This eliminates contradictions between recipe content and installed agent skills. - Add goal.md to all 21 recipes, 6 examples, and 5 cookbooks - Agent prompt builder uses goalOnly() — sends goal when present, falls back to full content for backward compat - Cookbook composition gains mode parameter: "agent" (goals only) vs "human" (full inline content) - Update intent blocks to direct agents to use skills for implementation - Update preamble text to reference goals instead of step-by-step content - Rename example content.md -> goal.md (already outcome-only) - Rename cookbook intro.md -> goal.md - Delete redundant rag-chat/deployment.md Co-authored-by: Isaac --- api/content-markdown.ts | 16 ++- .../ai-chat-app/{intro.md => goal.md} | 0 content/cookbooks/app-with-lakebase/goal.md | 8 ++ content/cookbooks/genie-analytics-app/goal.md | 7 + .../cookbooks/lakebase-off-platform/goal.md | 9 ++ .../operational-data-analytics/goal.md | 10 ++ .../{content.md => goal.md} | 0 .../content-moderator/{content.md => goal.md} | 0 .../{content.md => goal.md} | 0 content/examples/rag-chat/deployment.md | 25 ---- .../examples/rag-chat/{content.md => goal.md} | 0 .../saas-tracker/{content.md => goal.md} | 0 .../vacation-rentals/{content.md => goal.md} | 0 content/intent-cookbook.md | 15 +- content/intent-recipe.md | 17 +-- content/recipes/ai-chat-model-serving/goal.md | 10 ++ content/recipes/embeddings-generation/goal.md | 9 ++ content/recipes/foundation-models-api/goal.md | 9 ++ .../genie-conversational-analytics/goal.md | 10 ++ content/recipes/genie-multi-space/goal.md | 10 ++ content/recipes/lakebase-agent-memory/goal.md | 10 ++ .../goal.md | 9 ++ .../recipes/lakebase-create-instance/goal.md | 9 ++ .../recipes/lakebase-data-persistence/goal.md | 10 ++ .../lakebase-drizzle-off-platform/goal.md | 10 ++ .../goal.md | 10 ++ content/recipes/lakebase-pgvector/goal.md | 10 ++ .../recipes/lakebase-token-management/goal.md | 9 ++ .../medallion-architecture-from-cdc/goal.md | 10 ++ .../model-serving-endpoint-creation/goal.md | 9 ++ .../recipes/onboard-your-coding-agent/goal.md | 10 ++ .../set-up-your-local-dev-environment/goal.md | 9 ++ .../recipes/spin-up-databricks-app/goal.md | 9 ++ .../recipes/sync-tables-autoscaling/goal.md | 9 ++ content/recipes/unity-catalog-setup/goal.md | 10 ++ content/recipes/volume-file-upload/goal.md | 10 ++ ...8-agent-skills-unification-requirements.md | 130 ++++++++++++++++++ plugins/about-devhub.ts | 4 +- plugins/content-entries.ts | 28 ++-- scripts/validate-content.mjs | 22 +-- src/components/examples/example-detail.tsx | 4 +- src/lib/content-markdown.ts | 30 +++- src/lib/content-sections.ts | 26 +++- src/lib/cookbook-composition.ts | 40 ++++-- src/lib/copy-preamble.ts | 4 +- src/pages/templates/ai-chat-app.tsx | 2 +- tests/bootstrap-prompt.test.ts | 10 +- tests/markdown.test.ts | 48 +++---- tests/validate-content.test.ts | 2 +- 49 files changed, 537 insertions(+), 121 deletions(-) rename content/cookbooks/ai-chat-app/{intro.md => goal.md} (100%) create mode 100644 content/cookbooks/app-with-lakebase/goal.md create mode 100644 content/cookbooks/genie-analytics-app/goal.md create mode 100644 content/cookbooks/lakebase-off-platform/goal.md create mode 100644 content/cookbooks/operational-data-analytics/goal.md rename content/examples/agentic-support-console/{content.md => goal.md} (100%) rename content/examples/content-moderator/{content.md => goal.md} (100%) rename content/examples/inventory-intelligence/{content.md => goal.md} (100%) delete mode 100644 content/examples/rag-chat/deployment.md rename content/examples/rag-chat/{content.md => goal.md} (100%) rename content/examples/saas-tracker/{content.md => goal.md} (100%) rename content/examples/vacation-rentals/{content.md => goal.md} (100%) create mode 100644 content/recipes/ai-chat-model-serving/goal.md create mode 100644 content/recipes/embeddings-generation/goal.md create mode 100644 content/recipes/foundation-models-api/goal.md create mode 100644 content/recipes/genie-conversational-analytics/goal.md create mode 100644 content/recipes/genie-multi-space/goal.md create mode 100644 content/recipes/lakebase-agent-memory/goal.md create mode 100644 content/recipes/lakebase-change-data-feed-autoscaling/goal.md create mode 100644 content/recipes/lakebase-create-instance/goal.md create mode 100644 content/recipes/lakebase-data-persistence/goal.md create mode 100644 content/recipes/lakebase-drizzle-off-platform/goal.md create mode 100644 content/recipes/lakebase-off-platform-env-management/goal.md create mode 100644 content/recipes/lakebase-pgvector/goal.md create mode 100644 content/recipes/lakebase-token-management/goal.md create mode 100644 content/recipes/medallion-architecture-from-cdc/goal.md create mode 100644 content/recipes/model-serving-endpoint-creation/goal.md create mode 100644 content/recipes/onboard-your-coding-agent/goal.md create mode 100644 content/recipes/set-up-your-local-dev-environment/goal.md create mode 100644 content/recipes/spin-up-databricks-app/goal.md create mode 100644 content/recipes/sync-tables-autoscaling/goal.md create mode 100644 content/recipes/unity-catalog-setup/goal.md create mode 100644 content/recipes/volume-file-upload/goal.md create mode 100644 docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md diff --git a/api/content-markdown.ts b/api/content-markdown.ts index c50349c..16a1ec3 100644 --- a/api/content-markdown.ts +++ b/api/content-markdown.ts @@ -10,9 +10,10 @@ import { hasContentSlug, hasSolutionSlug, readContentSections, + readCookbookGoal, readCookbookIntro, } from "../src/lib/content-markdown"; -import { joinContentSections } from "../src/lib/content-sections"; +import { goalOnly } from "../src/lib/content-sections"; import { buildCookbookMarkdownDocument } from "../src/lib/cookbook-composition"; import { expandMdxImports } from "../src/lib/expand-mdx"; import { @@ -168,7 +169,7 @@ function readRecipeMarkdown(rootDir: string, slug: string): string { if (!hasContentSlug(rootDir, "recipes", slug)) { throw new Error(`Recipe page not found: "${slug}"`); } - return joinContentSections(readContentSections(rootDir, "recipes", slug)); + return goalOnly(readContentSections(rootDir, "recipes", slug)); } function readExampleMarkdown(rootDir: string, slug: string): string { @@ -176,9 +177,7 @@ function readExampleMarkdown(rootDir: string, slug: string): string { throw new Error(`Example page not found: "${slug}"`); } - const content = joinContentSections( - readContentSections(rootDir, "examples", slug), - ); + const content = goalOnly(readContentSections(rootDir, "examples", slug)); const example = examples.find((e) => e.id === slug); if (!example) { @@ -230,11 +229,14 @@ function readCookbookMarkdown(rootDir: string, slug: string): string { }; }); + const goal = readCookbookGoal(rootDir, slug); return buildCookbookMarkdownDocument({ cookbookName: cookbook.name, cookbookDescription: cookbook.description, - intro: readCookbookIntro(rootDir, slug), + goal, + intro: goal ? undefined : readCookbookIntro(rootDir, slug), recipes: recipeInputs, + mode: "agent", }); } @@ -345,7 +347,7 @@ export function loadAgentPromptParts( intentRecipe: readContent("intent-recipe"), intentCookbook: readContent("intent-cookbook"), intentExample: readContent("intent-example"), - localBootstrap: joinContentSections( + localBootstrap: goalOnly( readContentSections(rootDir, "recipes", LOCAL_BOOTSTRAP_SLUG), ), }; diff --git a/content/cookbooks/ai-chat-app/intro.md b/content/cookbooks/ai-chat-app/goal.md similarity index 100% rename from content/cookbooks/ai-chat-app/intro.md rename to content/cookbooks/ai-chat-app/goal.md diff --git a/content/cookbooks/app-with-lakebase/goal.md b/content/cookbooks/app-with-lakebase/goal.md new file mode 100644 index 0000000..06d0adb --- /dev/null +++ b/content/cookbooks/app-with-lakebase/goal.md @@ -0,0 +1,8 @@ +## What you are building + +A Databricks App with Lakebase Postgres for persistent data storage. The app has schema setup, full CRUD API routes, and deploys to the Databricks Apps platform. + +### Components + +1. **Create a Lakebase Instance** — provision a managed Postgres project with an endpoint and database, and collect the connection values. +2. **Lakebase Data Persistence** — add the Lakebase plugin to your app with schema initialization, CRUD routes, and data access patterns. diff --git a/content/cookbooks/genie-analytics-app/goal.md b/content/cookbooks/genie-analytics-app/goal.md new file mode 100644 index 0000000..ef90d00 --- /dev/null +++ b/content/cookbooks/genie-analytics-app/goal.md @@ -0,0 +1,7 @@ +## What you are building + +A minimal Databricks App with AI/BI Genie conversational analytics. Users ask natural-language questions about their data and get SQL-powered answers through an embedded Genie chat interface. + +### Components + +1. **Genie Conversational Analytics** — configure a Genie space, wire up the server and client plugins, declare app resources, and deploy. diff --git a/content/cookbooks/lakebase-off-platform/goal.md b/content/cookbooks/lakebase-off-platform/goal.md new file mode 100644 index 0000000..6e0bffd --- /dev/null +++ b/content/cookbooks/lakebase-off-platform/goal.md @@ -0,0 +1,9 @@ +## What you are building + +A connection from an app hosted outside the Databricks Apps platform (for example on AWS, Vercel, or Netlify) to Lakebase Postgres. The app uses portable environment configuration, token management with automatic credential refresh, and Drizzle ORM for type-safe database access. + +### Components + +1. **Lakebase Environment Management** — set up a Zod-validated environment configuration for secure Lakebase connection values. +2. **Lakebase Token Management** — implement token fetch, cache, and automatic refresh for Lakebase Postgres credentials. +3. **Drizzle ORM with Lakebase** — configure a Drizzle ORM pool with auto-refreshing credentials and migration support. diff --git a/content/cookbooks/operational-data-analytics/goal.md b/content/cookbooks/operational-data-analytics/goal.md new file mode 100644 index 0000000..17cf873 --- /dev/null +++ b/content/cookbooks/operational-data-analytics/goal.md @@ -0,0 +1,10 @@ +## What you are building + +An end-to-end operational data analytics pipeline: data flows from an OLTP database (Lakebase Postgres) through CDC replication into Unity Catalog, gets transformed through a medallion architecture (bronze/silver/gold layers), and is ready for dashboards and downstream consumers. + +### Components + +1. **Unity Catalog Setup** — configure Unity Catalog with external S3 storage for your destination catalog and schema. +2. **Create a Lakebase Instance** — provision a managed Postgres project as the OLTP source. +3. **Lakehouse Sync CDC** — enable change data capture replication from Lakebase tables to Unity Catalog Delta history tables. +4. **Medallion Architecture from CDC** — build silver (current-state) and gold (analytical) layers from the CDC history tables using Lakeflow Declarative Pipelines. diff --git a/content/examples/agentic-support-console/content.md b/content/examples/agentic-support-console/goal.md similarity index 100% rename from content/examples/agentic-support-console/content.md rename to content/examples/agentic-support-console/goal.md diff --git a/content/examples/content-moderator/content.md b/content/examples/content-moderator/goal.md similarity index 100% rename from content/examples/content-moderator/content.md rename to content/examples/content-moderator/goal.md diff --git a/content/examples/inventory-intelligence/content.md b/content/examples/inventory-intelligence/goal.md similarity index 100% rename from content/examples/inventory-intelligence/content.md rename to content/examples/inventory-intelligence/goal.md diff --git a/content/examples/rag-chat/deployment.md b/content/examples/rag-chat/deployment.md deleted file mode 100644 index 7e63841..0000000 --- a/content/examples/rag-chat/deployment.md +++ /dev/null @@ -1,25 +0,0 @@ -### 4. Install and deploy - -`databricks apps init` already wrote `.env` with the resolved Lakebase connection details. For a deploy-only flow you can go straight to deploy — `DATABRICKS_WORKSPACE_ID` and the Lakebase variables are auto-injected into the deployed runtime from `app.yaml` and the bound `postgres` resource. - -```bash -cd rag-chat-app -npm install -npm run deploy -``` - -`npm run deploy` wraps three steps: hydrate the bundle variable overrides from `.env` + the Lakebase Postgres API (`scripts/sync-bundle-vars.mjs`), `databricks bundle deploy` (creates the Databricks app on first run), and `databricks bundle run app` (starts it and prints the URL). - -#### Optional — run locally before deploying - -Local `npm run dev` needs `DATABRICKS_WORKSPACE_ID` (the **numeric** id used to build the AI Gateway URL) in `.env`. In the deployed app this is auto-injected; locally you have to fetch and patch it yourself: - -```bash -WORKSPACE_ID=$(databricks api get /api/2.1/unity-catalog/current-metastore-assignment \ - | python3 -c "import json,sys;print(json.load(sys.stdin)['workspace_id'])") -sed -i.bak "s/^DATABRICKS_WORKSPACE_ID=.*/DATABRICKS_WORKSPACE_ID=$WORKSPACE_ID/" .env && rm .env.bak - -npm run dev -``` - -(Optionally override `DATABRICKS_ENDPOINT` / `DATABRICKS_EMBEDDING_ENDPOINT` in `.env` if you want different chat / embeddings endpoints — also applies to deploy via `app.yaml`.) diff --git a/content/examples/rag-chat/content.md b/content/examples/rag-chat/goal.md similarity index 100% rename from content/examples/rag-chat/content.md rename to content/examples/rag-chat/goal.md diff --git a/content/examples/saas-tracker/content.md b/content/examples/saas-tracker/goal.md similarity index 100% rename from content/examples/saas-tracker/content.md rename to content/examples/saas-tracker/goal.md diff --git a/content/examples/vacation-rentals/content.md b/content/examples/vacation-rentals/goal.md similarity index 100% rename from content/examples/vacation-rentals/content.md rename to content/examples/vacation-rentals/goal.md diff --git a/content/intent-cookbook.md b/content/intent-cookbook.md index 8225735..eda0e71 100644 --- a/content/intent-cookbook.md +++ b/content/intent-cookbook.md @@ -2,22 +2,23 @@ The user copied the prompt for a DevHub **cookbook** — **{{name}}** ({{url}}). -A cookbook is a step-by-step pattern guide that walks the user through building an **archetype application** end-to-end on Databricks. Cookbooks are composed from multiple recipes — they show how the recipes fit together into a working app (e.g. an AI chat app with persistence, a Lakebase-backed CRUD app, a RAG chat app). The cookbook is the recommended starting point when the user wants the whole archetype, not just one piece. +A cookbook is a composed pattern that builds an **archetype application** end-to-end on Databricks from multiple recipe goals. The cookbook goal below describes the overall app and its components. Your installed Databricks agent skills contain the implementation patterns for each component. + +Use the cookbook goal for scope and architecture; use the skills for implementation. Your job in this conversation is to: 1. Clarify the user's **goal for this archetype** — production app, learning project, or demo. 2. Verify the local Databricks dev environment is ready (block below). -3. Walk the user through the cookbook section by section, asking the questions each section surfaces, and stitching the included recipes together coherently. -4. When the cookbook content and your installed Databricks agent skills cover the same topic, **treat the skills as the source of truth** for implementation patterns, CLI commands, and code. The cookbook provides context and scope; the skills provide the authoritative how-to. +3. Use the component goals to understand scope, then **use your installed Databricks agent skills** to implement each component step by step. ## Step 1 — Clarify intent before touching code Ask **one** question, ideally with a multiple-choice tool: -- **New project from scratch** following this archetype end-to-end. → Run the local-bootstrap below, then scaffold a fresh project and walk through the cookbook step by step. +- **New project from scratch** following this archetype end-to-end. → Run the local-bootstrap below, then scaffold a fresh project and work through each component. - **Add this archetype to an existing Databricks app**. → Read the user's existing project first; introduce the archetype's pieces incrementally without breaking what's there. -- **Just learning the pattern**: the user wants to understand the archetype before deciding to build it. → Walk through the steps as a guided tour; do not execute commands. +- **Just learning the pattern**: the user wants to understand the archetype before deciding to build it. → Walk through the component goals as a guided tour; do not execute commands. - **Not sure — help me decide**: ask follow-ups about the user's end goal (who uses the app, what data, deployed where) and map back to one of the above. ## Step 2 — Pin down archetype-specific decisions @@ -31,6 +32,6 @@ Cookbooks compose multiple Databricks primitives — Lakebase, Agent Bricks, Mod ## Step 3 — Verify the local Databricks dev environment -Cookbooks run multiple `databricks` and AppKit CLI commands across their steps; a misconfigured CLI profile fails immediately and looks like a cookbook bug. **Walk the user through the local-bootstrap block below first**, even if they say their environment is already set up. +Cookbooks run multiple CLI and AppKit commands across their components; a misconfigured CLI profile fails immediately and looks like a cookbook bug. **Walk the user through the local-bootstrap block below first**, even if they say their environment is already set up. -The full cookbook content the user is focused on is attached after the local-bootstrap block. +The cookbook goal and component goals are attached after the local-bootstrap block. diff --git a/content/intent-recipe.md b/content/intent-recipe.md index 145cadf..3127a13 100644 --- a/content/intent-recipe.md +++ b/content/intent-recipe.md @@ -4,25 +4,26 @@ The user copied the prompt for a DevHub **recipe** — **{{name}}** ({{url}}). A recipe is a focused, opinionated how-to for a single Databricks pattern (e.g. wiring Lakebase Change Data Feed, creating a Model Serving endpoint, persisting chat history). Recipes are designed to be dropped into an existing project or composed into a larger build. They are deliberately narrow — they solve one thing well. +The recipe goal below describes **what** the user wants to build — the outcome and key decisions. Your installed Databricks agent skills contain the **how** — the implementation patterns, CLI commands, and code. Use the goal for scope and context; use the skills for implementation. + Your job in this conversation is to: 1. Clarify whether the user is **integrating this recipe into an existing project** or **starting fresh from scratch**, and adapt accordingly. 2. Verify the local Databricks dev environment is ready (block below). -3. Walk the user through the recipe step by step, asking the questions the recipe itself surfaces. -4. When the recipe content and your installed Databricks agent skills cover the same topic, **treat the skills as the source of truth** for implementation patterns, CLI commands, and code. The recipe provides context and scope; the skills provide the authoritative how-to. +3. Use the recipe goal to understand scope, then **use your installed Databricks agent skills** to implement it step by step. ## Step 1 — Clarify intent before touching code Ask **one** question, ideally with a multiple-choice tool (see guidelines): -- **Existing project**: the user already has a Databricks app / repo and wants to add this pattern to it. → Read the user's existing project structure first; the recipe steps will be applied surgically. -- **New project from this recipe**: the user wants this recipe as the starting point of a new app. → Run the local-bootstrap below first, then follow the recipe. -- **Just learning**: the user wants to read through the recipe and understand it without building anything yet. → Walk through the steps as a tutorial; do not execute commands. +- **Existing project**: the user already has a Databricks app / repo and wants to add this pattern to it. → Read the user's existing project structure first; apply the pattern surgically. +- **New project from this recipe**: the user wants this recipe as the starting point of a new app. → Run the local-bootstrap below first, then implement. +- **Just learning**: the user wants to understand the pattern without building anything yet. → Walk through the goal and explain what each piece does. - **Not sure — help me decide**: ask the user what they're trying to accomplish at the project level, then map back to one of the above. ## Step 2 — Pin down recipe-specific decisions -Once the integration mode is clear, ask any follow-ups the recipe itself surfaces — typically about which Databricks resources to use: +Once the integration mode is clear, ask any follow-ups — typically about which Databricks resources to use: - Should we **create new resources** (catalog, schema, Lakebase instance, serving endpoint) or **reuse existing ones** the user already has? Never assume; always ask. - Which **Databricks profile** should the CLI commands target? (`databricks auth profiles` to list valid profiles.) @@ -30,6 +31,6 @@ Once the integration mode is clear, ask any follow-ups the recipe itself surface ## Step 3 — Verify the local Databricks dev environment -Whether integrating or starting fresh, the recipe's commands assume a working Databricks CLI profile and (for app-related recipes) an AppKit project. **Walk the user through the local-bootstrap block below before running any recipe commands** — even if they think the environment is already set up, the verification steps are quick and prevent confusing failures downstream. +Whether integrating or starting fresh, your skills' commands assume a working Databricks CLI profile and (for app-related recipes) an AppKit project. **Walk the user through the local-bootstrap block below before running any commands** — even if they think the environment is already set up. -The full recipe content the user is focused on is attached after the local-bootstrap block. +The recipe goal the user is focused on is attached after the local-bootstrap block. diff --git a/content/recipes/ai-chat-model-serving/goal.md b/content/recipes/ai-chat-model-serving/goal.md new file mode 100644 index 0000000..40ccc98 --- /dev/null +++ b/content/recipes/ai-chat-model-serving/goal.md @@ -0,0 +1,10 @@ +## Streaming AI Chat with Model Serving + +Build a streaming AI chat experience in a Databricks App using Databricks Model Serving with OpenAI-compatible endpoints. + +When done, you will have: + +- A real-time streaming chat interface in your Databricks App +- Integration with Databricks Model Serving via AI Gateway +- Server-side chat transport and client-side chat UI wired together +- A deployed app where users can converse with a Databricks-hosted LLM diff --git a/content/recipes/embeddings-generation/goal.md b/content/recipes/embeddings-generation/goal.md new file mode 100644 index 0000000..6a21689 --- /dev/null +++ b/content/recipes/embeddings-generation/goal.md @@ -0,0 +1,9 @@ +## Generate Embeddings with AI Gateway + +Generate text embeddings from a Databricks AI Gateway endpoint using the Databricks SDK. + +When done, you will have: + +- A configured embedding endpoint in your app environment +- A reusable helper function that generates vector embeddings from text input +- Integration with your existing Databricks workspace client via AppKit diff --git a/content/recipes/foundation-models-api/goal.md b/content/recipes/foundation-models-api/goal.md new file mode 100644 index 0000000..c014b93 --- /dev/null +++ b/content/recipes/foundation-models-api/goal.md @@ -0,0 +1,9 @@ +## Query AI Gateway Endpoints + +Access Databricks foundation models through AI Gateway endpoints with built-in governance, monitoring, and streaming support. + +When done, you will have: + +- A configured AI Gateway endpoint for chat inference in your app +- A query helper using the Databricks SDK for standard request/response interactions +- An optional streaming setup using the Vercel AI SDK for real-time chat responses diff --git a/content/recipes/genie-conversational-analytics/goal.md b/content/recipes/genie-conversational-analytics/goal.md new file mode 100644 index 0000000..eac2146 --- /dev/null +++ b/content/recipes/genie-conversational-analytics/goal.md @@ -0,0 +1,10 @@ +## Genie Conversational Analytics + +Embed a Databricks AI/BI Genie chat interface in your app so users can explore data through natural language. + +When done, you will have: + +- A configured AI/BI Genie space connected to your data tables +- A Databricks App with an embedded Genie chat interface +- Server and client plugins wired together with proper app resource declarations +- A deployed app where users can ask questions about their data in plain language diff --git a/content/recipes/genie-multi-space/goal.md b/content/recipes/genie-multi-space/goal.md new file mode 100644 index 0000000..e236418 --- /dev/null +++ b/content/recipes/genie-multi-space/goal.md @@ -0,0 +1,10 @@ +## Genie Multi-Space Selector + +Upgrade a single-space Genie app to let users switch between multiple AI/BI Genie spaces from a dropdown. + +When done, you will have: + +- Multiple Genie spaces accessible from a single app +- A dropdown selector for switching between named Genie spaces +- Automatic conversation state cleanup when switching spaces +- Server and client configuration supporting multiple space aliases diff --git a/content/recipes/lakebase-agent-memory/goal.md b/content/recipes/lakebase-agent-memory/goal.md new file mode 100644 index 0000000..382f76f --- /dev/null +++ b/content/recipes/lakebase-agent-memory/goal.md @@ -0,0 +1,10 @@ +## Lakebase Agent Memory + +Persist AI agent chat conversations to Lakebase so users can resume sessions, view full message history, and let the agent reason over previous turns across requests and deploys. + +When done, you will have: + +- A relational schema (chats and messages tables) in Lakebase for storing conversations +- Durable persistence of every chat turn: user input, assistant replies, and tool calls +- An app where users can return to previous chat sessions and continue where they left off +- Agent memory that survives restarts, deploys, and machine changes diff --git a/content/recipes/lakebase-change-data-feed-autoscaling/goal.md b/content/recipes/lakebase-change-data-feed-autoscaling/goal.md new file mode 100644 index 0000000..e66000f --- /dev/null +++ b/content/recipes/lakebase-change-data-feed-autoscaling/goal.md @@ -0,0 +1,9 @@ +## Replicate Lakebase Tables to Unity Catalog with Lakehouse Sync + +Replicate Lakebase Autoscaling Postgres tables into Unity Catalog as managed Delta tables using Lakehouse Sync, capturing every row-level change as SCD Type 2 history. + +When done, you will have: + +- Delta history tables in Unity Catalog with full change tracking for every insert, update, and delete +- Continuous CDC replication from Lakebase Postgres to the lakehouse with no external compute +- Operational data queryable in Spark SQL, notebooks, BI tools, and downstream pipelines diff --git a/content/recipes/lakebase-create-instance/goal.md b/content/recipes/lakebase-create-instance/goal.md new file mode 100644 index 0000000..c4f6b2f --- /dev/null +++ b/content/recipes/lakebase-create-instance/goal.md @@ -0,0 +1,9 @@ +## Create a Lakebase Instance + +Provision a managed Lakebase Postgres project on Databricks and collect the connection values needed by downstream recipes. + +When done, you will have: + +- A managed Postgres cluster running in your Databricks workspace +- A production branch with an active endpoint and default database +- Connection values (host, endpoint path, database path, database name) ready for use in other Lakebase recipes diff --git a/content/recipes/lakebase-data-persistence/goal.md b/content/recipes/lakebase-data-persistence/goal.md new file mode 100644 index 0000000..11c5d54 --- /dev/null +++ b/content/recipes/lakebase-data-persistence/goal.md @@ -0,0 +1,10 @@ +## Lakebase Data Persistence + +Add a managed Postgres database to your Databricks App using the Lakebase plugin, with schema setup, table creation, and full CRUD REST API routes. + +When done, you will have: + +- A Databricks App connected to a Lakebase Postgres database +- Database schema and tables for your domain entities +- Working CRUD API routes (create, read, update, delete) backed by Lakebase +- A deployed app with persistent data storage diff --git a/content/recipes/lakebase-drizzle-off-platform/goal.md b/content/recipes/lakebase-drizzle-off-platform/goal.md new file mode 100644 index 0000000..da77798 --- /dev/null +++ b/content/recipes/lakebase-drizzle-off-platform/goal.md @@ -0,0 +1,10 @@ +## Drizzle ORM with Lakebase in an Off-Platform App + +Connect Drizzle ORM to Lakebase in any Node.js server outside Databricks App Platform, with automatic credential refresh and migration support. + +When done, you will have: + +- A Lakebase-backed connection pool with automatic token refresh via password callback +- Drizzle ORM initialized with your schema and ready for type-safe queries +- A migration script that builds a temporary connection URL with fresh Lakebase credentials +- A working Drizzle Kit configuration for schema generation and migrations diff --git a/content/recipes/lakebase-off-platform-env-management/goal.md b/content/recipes/lakebase-off-platform-env-management/goal.md new file mode 100644 index 0000000..cb321c5 --- /dev/null +++ b/content/recipes/lakebase-off-platform-env-management/goal.md @@ -0,0 +1,10 @@ +## Lakebase Environment Management for Off-Platform Apps + +Define and validate the environment variables needed to connect to Lakebase from apps deployed outside Databricks App Platform. + +When done, you will have: + +- All Lakebase connection values collected from the Databricks CLI +- A Zod-based environment validation module that fails fast on missing or invalid variables +- Support for both token auth (local dev) and M2M OAuth (production) +- An `.env.example` file documenting every required variable for your team and CI diff --git a/content/recipes/lakebase-pgvector/goal.md b/content/recipes/lakebase-pgvector/goal.md new file mode 100644 index 0000000..6906cd5 --- /dev/null +++ b/content/recipes/lakebase-pgvector/goal.md @@ -0,0 +1,10 @@ +## Vector Search with Lakebase and pgvector + +Enable vector similarity search in your Lakebase Postgres database using the pgvector extension, with a server-side module for storing and querying embeddings. + +When done, you will have: + +- The pgvector extension enabled on your Lakebase instance +- A vector embedding table with configurable dimensions +- Server-side functions for inserting documents and performing similarity search +- An IVFFlat or HNSW index for efficient nearest-neighbor queries diff --git a/content/recipes/lakebase-token-management/goal.md b/content/recipes/lakebase-token-management/goal.md new file mode 100644 index 0000000..1aa0fc2 --- /dev/null +++ b/content/recipes/lakebase-token-management/goal.md @@ -0,0 +1,9 @@ +## Lakebase Token Management + +Fetch, cache, and automatically refresh the short-lived Postgres credentials that Lakebase requires, supporting both token auth and M2M OAuth. + +When done, you will have: + +- A token manager that fetches and caches Lakebase Postgres credentials with automatic refresh before expiry +- Support for direct token auth (local development) and M2M OAuth (production) +- A local dev script that refreshes your workspace token in your env file diff --git a/content/recipes/medallion-architecture-from-cdc/goal.md b/content/recipes/medallion-architecture-from-cdc/goal.md new file mode 100644 index 0000000..a4e58b0 --- /dev/null +++ b/content/recipes/medallion-architecture-from-cdc/goal.md @@ -0,0 +1,10 @@ +## Medallion Architecture from CDC History Tables + +Transform Lakehouse Sync CDC history tables into a layered medallion architecture with bronze, silver, and gold layers using Lakeflow Declarative Pipelines. + +When done, you will have: + +- A silver layer with deduplicated, current-state materialized views for each entity +- A gold layer with business aggregations and metrics as materialized views +- A scheduled Lakeflow Declarative Pipeline refreshing silver and gold layers incrementally +- All layers queryable as Unity Catalog tables via SQL, Spark, BI tools, and Genie diff --git a/content/recipes/model-serving-endpoint-creation/goal.md b/content/recipes/model-serving-endpoint-creation/goal.md new file mode 100644 index 0000000..463a5e9 --- /dev/null +++ b/content/recipes/model-serving-endpoint-creation/goal.md @@ -0,0 +1,9 @@ +## Create a Databricks Model Serving Endpoint + +Provision and validate a Databricks Model Serving endpoint for AI chat inference. + +When done, you will have: + +- A model serving endpoint running in your Databricks workspace +- The endpoint tested and confirmed ready for inference requests +- The endpoint name configured in your app for local development and deployment diff --git a/content/recipes/onboard-your-coding-agent/goal.md b/content/recipes/onboard-your-coding-agent/goal.md new file mode 100644 index 0000000..b835092 --- /dev/null +++ b/content/recipes/onboard-your-coding-agent/goal.md @@ -0,0 +1,10 @@ +## Onboard Your Coding Agent + +Make a Databricks repo agent-ready so your coding agent understands the Databricks platform and can fetch DevHub documentation on demand. + +When done, you will have: + +- Databricks platform skills installed at project scope, traveling with the repo +- The DevHub Docs MCP server wired up for on-demand access to any DevHub page +- An AGENTS.md file (with symlinked CLAUDE.md) pinning workspace defaults for the codebase +- A coding agent that generates correct Databricks code instead of guessing diff --git a/content/recipes/set-up-your-local-dev-environment/goal.md b/content/recipes/set-up-your-local-dev-environment/goal.md new file mode 100644 index 0000000..bf2ede7 --- /dev/null +++ b/content/recipes/set-up-your-local-dev-environment/goal.md @@ -0,0 +1,9 @@ +## Set Up Your Local Dev Environment + +Install the Databricks CLI, authenticate a profile, and verify the handshake. Every other DevHub template assumes this has already passed. + +After this step you will have: + +- Databricks CLI `0.296+` installed and on `PATH` +- An authenticated CLI profile (`databricks auth profiles` shows `Valid: YES`) +- A successful smoke test (`databricks current-user me` returns your identity) diff --git a/content/recipes/spin-up-databricks-app/goal.md b/content/recipes/spin-up-databricks-app/goal.md new file mode 100644 index 0000000..dedd623 --- /dev/null +++ b/content/recipes/spin-up-databricks-app/goal.md @@ -0,0 +1,9 @@ +## Spin Up a Databricks App + +Generate a working AppKit Databricks App from scratch and deploy it to your workspace. + +When done, you will have: + +- A scaffolded Databricks App project with your chosen plugins (Lakebase, analytics, Genie, model serving) +- A locally running development server for testing +- A deployed app accessible in your Databricks workspace diff --git a/content/recipes/sync-tables-autoscaling/goal.md b/content/recipes/sync-tables-autoscaling/goal.md new file mode 100644 index 0000000..a83bd28 --- /dev/null +++ b/content/recipes/sync-tables-autoscaling/goal.md @@ -0,0 +1,9 @@ +## Sync a Unity Catalog Table to Lakebase + +Serve lakehouse data through Lakebase Autoscaling Postgres so your applications can query it with sub-10ms latency using a synced table that stays up to date automatically. + +When done, you will have: + +- A synced table in Unity Catalog tracking the replication pipeline +- A read-only Postgres table in Lakebase queryable with sub-10ms latency from any standard Postgres client +- A managed Lakeflow pipeline keeping the data in sync via snapshot, triggered, or continuous mode diff --git a/content/recipes/unity-catalog-setup/goal.md b/content/recipes/unity-catalog-setup/goal.md new file mode 100644 index 0000000..c1c23d3 --- /dev/null +++ b/content/recipes/unity-catalog-setup/goal.md @@ -0,0 +1,10 @@ +## Set Up Unity Catalog with External Storage + +Create a Unity Catalog catalog backed by an external S3 bucket for scenarios that require custom storage control. + +When done, you will have: + +- An IAM role granting Databricks access to your S3 bucket +- A storage credential and external location registered in Unity Catalog +- A Unity Catalog catalog using your external S3 bucket as its storage root +- Infrastructure ready for Sync Tables, cross-account access, or custom lifecycle policies diff --git a/content/recipes/volume-file-upload/goal.md b/content/recipes/volume-file-upload/goal.md new file mode 100644 index 0000000..9fe9161 --- /dev/null +++ b/content/recipes/volume-file-upload/goal.md @@ -0,0 +1,10 @@ +## Volume File Manager + +Add file upload, browsing, download, delete, file type validation, and CSV row preview to your Databricks App using Unity Catalog Volumes. + +When done, you will have: + +- A Unity Catalog Volume configured as file storage +- A file management UI with upload, browse, download, and delete capabilities +- File type validation and CSV row preview functionality +- Automatically registered HTTP routes for all file operations diff --git a/docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md b/docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md new file mode 100644 index 0000000..cfb4544 --- /dev/null +++ b/docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md @@ -0,0 +1,130 @@ +--- +date: 2026-05-18 +topic: agent-skills-unification +--- + +# Unifying Agent Skills with DevHub Templates + +## Problem Frame + +DevHub templates (recipes, cookbooks, examples) and agent skills both contain implementation guidance -- CLI commands, code patterns, configuration. The content overlaps significantly, causing: + +1. **Contradictions** -- recipe says CLI 0.296+, skill says 0.292+. Agents receiving both must reconcile. +2. **Drift** -- when one side updates, the other gets stale. No sync mechanism exists. +3. **Bloated agent prompts** -- agents receive full step-by-step recipe content AND skills covering the same topic. Redundant tokens, conflicting instructions. + +PR #95 partially addressed this by adding "skills = source of truth" directives to intent blocks and converting examples to outcome-only. But recipes still send full implementation content to agents, creating the contradiction window. + +## Key Decisions + +- **Skills are the source of truth** for implementation (CLI commands, code patterns, configuration). Recipe content is authoritative for human-facing tutorials but not for agent prompts. +- **Agent-first does not mean human-hostile.** Humans keep full step-by-step instructions on the web page. The change is what agents receive, not what humans see. +- **Physical file boundary** separates agent content from human content. No section markers or parsing -- different files for different audiences. + +## Requirements + +**File Structure** + +- R1. All template types (recipes, cookbooks, examples) use `goal.md` for the outcome description. This file describes WHAT you'll build, key decisions, services involved, and what the result looks like. +- R2. Recipes have a second file `content.md` containing human-targeted step-by-step instructions. This includes prerequisites (folded in from current `prerequisites.md`) and implementation steps. +- R3. Cookbooks have only `goal.md` (replaces the current `intro.md`). It describes the overall composed app and how the pieces fit together. +- R4. Examples have only `goal.md` (renamed from current `content.md`). Examples are already outcome-only. +- R5. Delete all `prerequisites.md` files. Their content moves into the top of each recipe's `content.md`. + +**Agent Prompt Composition** + +- R6. Agent prompt for recipes includes only `goal.md` (not `content.md`). Skills handle implementation. +- R7. Agent prompt for cookbooks includes the cookbook's `goal.md` followed by each constituent recipe's `goal.md`. No recipe `content.md` is sent. +- R8. Agent prompt for examples includes only `goal.md` (no behavioral change since examples are already outcome-only, just reads from the renamed file). +- R9. The local bootstrap recipe (`set-up-your-local-dev-environment`) agent prompt also sends only `goal.md`. Skills and the dev-guidelines preamble handle environment verification. + +**Human Page Rendering** + +- R10. Recipe detail pages render `goal.md` + `content.md` joined together. The outcome description appears at the top, followed by the full step-by-step instructions. Visual separation uses existing `prose-solution` CSS heading borders (h2 gets `border-b` automatically) -- no new components needed. +- R11. Cookbook detail pages keep inline composition: cookbook `goal.md` at top, then each recipe's `goal.md` + `content.md` inlined (same reading experience as today). +- R12. Example detail pages render `goal.md` (no behavioral change, just reads from the renamed file). + +**Cookbook Composition** + +- R13. `composeCookbookMarkdown()` gains an optional mode parameter (defaulting to current behavior) to compose agent-only output (goals only) vs human output (goals + content). Single function with mode parameter preferred over two separate functions to avoid duplicating ~40 lines of shared composition logic. +- R14. All 5 cookbooks get a `goal.md`. The existing `ai-chat-app/intro.md` becomes `ai-chat-app/goal.md`. The other 4 cookbooks need new `goal.md` files written. +- R20. Cookbook `goal.md` explicitly frames the overall outcome as composed of multiple components and lists the constituent recipe goals. The agent prompt composition function wraps each recipe's `goal.md` with a labeled heading (e.g., "Component: Set Up Model Serving") so agents understand the hierarchy -- big goal at the top, sub-goals below. + +**Content Migration** + +- R15. For each of the 21 recipes: extract the outcome description from current `content.md` into a new `goal.md`, leave the implementation in `content.md`, fold `prerequisites.md` content into the top of `content.md`. Effort breakdown: 8 recipes have clear outcome intros (easy extraction), 10 dive into steps (need 2-3 sentence outcomes written), 3 need significant writing (Lakehouse Sync CDC, Token Management, Sync Tables). +- R16. For each of the 6 examples: rename `content.md` to `goal.md`. +- R17. The `rag-chat` example's `deployment.md` is implementation-oriented (npm commands, env hydration, workspace ID injection). It should NOT fold into `goal.md`. Since the example points to a GitHub repo and `template/README.md` is the deployment runbook, delete `deployment.md` (it's redundant with the repo README). + +**Intent Blocks and Preamble** + +- R18. Update `content/intent-recipe.md` to reflect that the agent receives only the goal, not implementation steps. Remove references to "follow the recipe step by step" for the implementation path; instead direct agents to use installed skills. +- R19. Update `content/intent-cookbook.md` similarly -- the agent receives goals for each recipe, not full content. + +## Flow Diagram + +``` + Recipe folder + / \ + goal.md content.md + (outcome) (human instructions) + | | + +-------+-------+ | + | | | + Agent prompt Human page Human page + (goal only) (goal + content joined) + | + v + Installed skills + (implementation) +``` + +``` + Cookbook folder + | + goal.md + (overall outcome) + | + +--------+--------+ + | | + Agent prompt Human page + (cookbook goal + (cookbook goal + + recipe goals) recipe goals + + | recipe content + v inlined) + Installed skills +``` + +## Success Criteria + +- Agent prompts for recipes contain zero implementation details (no CLI commands, no code blocks, no version requirements). Only outcome descriptions and key decisions. +- Human recipe pages render identically to today's experience (same content, same structure, same reading flow) except the outcome section appears prominently at the top. +- No contradictions possible between recipe agent content and skills -- they cover different concerns (scope vs implementation). +- Cookbook agent prompts are significantly shorter (goal descriptions only, not full recipe bodies). + +## Scope Boundaries + +- **Not changing recipe content for humans.** The step-by-step instructions stay on the web page. We're changing what agents receive, not what humans read. +- **Not syncing content between repos.** Skills and recipe content are maintained independently. They serve different audiences with different formats. +- **Not changing the agent-skills repo.** The incorporate-devhub-recipes branch is a separate workstream. This PR focuses on the DevHub side. +- **Not changing the example template/GitHub repo pattern.** Examples already work well. + +## Dependencies / Assumptions + +- Agent skills are installed in the user's environment (dev-guidelines already requires this). +- The incorporate-devhub-recipes branch in the agent-skills repo lands independently. It already contains the implementation content that agents need. +- The `goal.md` content for each recipe needs to be written well enough that agents understand the scope without implementation details. + +## Resolved Questions + +All questions originally deferred to planning have been investigated and resolved: + +- **Local bootstrap (R9):** `databricks-core` skill fully covers CLI install (all OS paths), OAuth auth, profile verification, and smoke testing. Actually more comprehensive than the recipe. No special treatment needed. +- **Cookbook composition (R13):** Mode parameter on existing function. Avoids duplicating ~40 lines of shared logic. Backward-compatible default. +- **Recipe outcome boundaries (R15):** 8 recipes have clear outcome intros (easy extraction), 10 dive into steps (need short outcomes written), 3 need significant writing. +- **rag-chat deployment.md (R17):** Implementation-oriented, not outcome-oriented. Redundant with `template/README.md` in the GitHub repo. Delete it. +- **Visual separation (R10):** Existing `prose-solution` CSS heading borders provide natural separation. No new components. + +## Next Steps + +-> `/ce:plan` for structured implementation planning diff --git a/plugins/about-devhub.ts b/plugins/about-devhub.ts index 97676f1..4671d79 100644 --- a/plugins/about-devhub.ts +++ b/plugins/about-devhub.ts @@ -2,7 +2,7 @@ import { readFileSync } from "fs"; import { resolve } from "path"; import type { LoadContext, Plugin } from "@docusaurus/types"; import { ABOUT_DEVHUB_SLUG } from "../src/lib/bootstrap-prompt"; -import { joinContentSections } from "../src/lib/content-sections"; +import { goalOnly } from "../src/lib/content-sections"; import { readContentSections } from "../src/lib/content-markdown"; import type { AgentPromptParts } from "../src/lib/copy-preamble"; @@ -29,7 +29,7 @@ function readMarkdownFile(siteDir: string, slug: string): string { } function readLocalBootstrap(siteDir: string): string { - return joinContentSections( + return goalOnly( readContentSections(siteDir, "recipes", LOCAL_BOOTSTRAP_SLUG), ); } diff --git a/plugins/content-entries.ts b/plugins/content-entries.ts index bd1f781..89662b0 100644 --- a/plugins/content-entries.ts +++ b/plugins/content-entries.ts @@ -6,10 +6,7 @@ import { getSolutionSlugs, readContentSections, } from "../src/lib/content-markdown"; -import { - joinContentSections, - type ContentSections, -} from "../src/lib/content-sections"; +import { goalOnly, type ContentSections } from "../src/lib/content-sections"; import { routePathWithBaseUrl } from "../src/lib/site-paths"; import { recipes, @@ -53,12 +50,21 @@ function createFolderRouteModuleSource( sections: ContentSections, ): string { const section = entryType === "recipe" ? "recipes" : "examples"; + const hasGoal = sections.goal !== undefined; const hasPrereqs = sections.prerequisites !== undefined; const hasDeploy = sections.deployment !== undefined; - const imports: string[] = [ - `import Content from "@site/content/${section}/${slug}/content.md";`, - ]; + const imports: string[] = []; + if (hasGoal) { + imports.push( + `import Goal from "@site/content/${section}/${slug}/goal.md";`, + ); + } + if (sections.content !== undefined) { + imports.push( + `import Content from "@site/content/${section}/${slug}/content.md";`, + ); + } if (hasPrereqs) { imports.push( `import Prerequisites from "@site/content/${section}/${slug}/prerequisites.md";`, @@ -77,7 +83,11 @@ function createFolderRouteModuleSource( ? '

Deployment

\n ' : null; - const children = [prereqsBlock, " ", deployBlock] + const goalBlock = hasGoal ? " " : null; + const contentBlock = + sections.content !== undefined ? " " : null; + + const children = [goalBlock, prereqsBlock, contentBlock, deployBlock] .filter(Boolean) .join("\n"); @@ -214,7 +224,7 @@ export default function contentEntriesPlugin( slug, ); sectionsBySlug[slug] = sections; - rawMarkdownBySlug[slug] = joinContentSections(sections); + rawMarkdownBySlug[slug] = goalOnly(sections); } else { const filePath = resolve( context.siteDir, diff --git a/scripts/validate-content.mjs b/scripts/validate-content.mjs index 0d657b6..c285d00 100644 --- a/scripts/validate-content.mjs +++ b/scripts/validate-content.mjs @@ -12,14 +12,16 @@ if (!existsSync(resolve(ROOT, "content"))) { } const RESOURCE_ALLOWED_FILES = new Set([ + "goal.md", "content.md", "prerequisites.md", "deployment.md", ]); -const RESOURCE_REQUIRED_FILE = "content.md"; +/** A folder must have at least one of these to be published. */ +const RESOURCE_REQUIRED_FILES = ["goal.md", "content.md"]; const RESOURCE_SECTIONS = /** @type {const} */ (["recipes", "examples"]); -const COOKBOOK_ALLOWED_FILES = new Set(["intro.md"]); +const COOKBOOK_ALLOWED_FILES = new Set(["goal.md", "intro.md"]); /** @type {string[]} */ const errors = []; @@ -33,7 +35,7 @@ const errors = []; * @param {string} opts.sectionPath e.g. "content/recipes" — used in error messages * @param {string} opts.sectionDir absolute filesystem path to the section * @param {Set} opts.allowedFiles whitelist of allowed direct-child filenames - * @param {string=} opts.requiredFile filename that must be present (omit for none) + * @param {string[]=} opts.requiredFiles at least one of these must be present (omit for none) * @param {string} opts.emptyHint trailing instruction appended to the "is empty" error * @param {string} opts.flatHint trailing instruction appended to the "is not a directory" error */ @@ -41,7 +43,7 @@ function validateContentFolder({ sectionPath, sectionDir, allowedFiles, - requiredFile, + requiredFiles, emptyHint, flatHint, }) { @@ -75,9 +77,13 @@ function validateContentFolder({ } } - if (requiredFile && !files.includes(requiredFile)) { + if ( + requiredFiles && + requiredFiles.length > 0 && + !requiredFiles.some((f) => files.includes(f)) + ) { errors.push( - `${sectionPath}/${entry}/ is missing the required ${requiredFile}.`, + `${sectionPath}/${entry}/ is missing a required file. Need at least one of: ${requiredFiles.join(", ")}.`, ); } } @@ -88,8 +94,8 @@ for (const section of RESOURCE_SECTIONS) { sectionPath: `content/${section}`, sectionDir: resolve(ROOT, "content", section), allowedFiles: RESOURCE_ALLOWED_FILES, - requiredFile: RESOURCE_REQUIRED_FILE, - emptyHint: "Add content.md.", + requiredFiles: RESOURCE_REQUIRED_FILES, + emptyHint: "Add goal.md or content.md.", flatHint: `Flat files are not allowed. Move to content/${section}//content.md.`, }); } diff --git a/src/components/examples/example-detail.tsx b/src/components/examples/example-detail.tsx index 1508ca7..f6c3233 100644 --- a/src/components/examples/example-detail.tsx +++ b/src/components/examples/example-detail.tsx @@ -23,7 +23,7 @@ import { import type { Example } from "@/lib/recipes/recipes"; import { cookbooks, recipes } from "@/lib/recipes/recipes"; import { useExampleSections } from "@/lib/use-raw-content-markdown"; -import { joinContentSections } from "@/lib/content-sections"; +import { goalOnly } from "@/lib/content-sections"; import { TemplateImageCarousel } from "@/components/examples/template-image-carousel"; import { TemplatePreviewImage } from "@/components/examples/template-preview-image"; import { FallbackCardArt } from "@/components/examples/fallback-card-art"; @@ -136,7 +136,7 @@ export function ExampleDetail({ const githubUrl = `${GITHUB_BASE}/${example.githubPath}/template`; const sections = useExampleSections(example.id) ?? { content: "" }; - const rawMarkdown = joinContentSections(sections); + const rawMarkdown = goalOnly(sections); const includedCookbooks = example.cookbookIds .map((id) => cookbooks.find((c) => c.id === id)) diff --git a/src/lib/content-markdown.ts b/src/lib/content-markdown.ts index cb12f2f..4e0c612 100644 --- a/src/lib/content-markdown.ts +++ b/src/lib/content-markdown.ts @@ -30,6 +30,10 @@ export function hasSolutionSlug(rootDir: string, slug: string): boolean { } /** Recipes and examples live in `content/
//` folders with a required content.md. */ +/** + * Recipes and examples live in `content/
//` folders. + * A folder is published if it has either goal.md or content.md (or both). + */ export function getContentSlugs( rootDir: string, section: FolderContentSection, @@ -39,8 +43,9 @@ export function getContentSlugs( .filter((entry) => { const fullPath = resolve(directory, entry); if (!statSync(fullPath).isDirectory()) return false; - return existsSync( - resolve(fullPath, `${REQUIRED_CONTENT_SECTION_FILE}.md`), + return ( + existsSync(resolve(fullPath, "goal.md")) || + existsSync(resolve(fullPath, `${REQUIRED_CONTENT_SECTION_FILE}.md`)) ); }) .sort(); @@ -97,16 +102,27 @@ export function readCookbookIntro( return readFileSync(filePath, "utf-8"); } -/** Reads all present section files; throws when the required content.md is missing. */ +/** Reads `content/cookbooks//goal.md` if present. */ +export function readCookbookGoal( + rootDir: string, + slug: string, +): string | undefined { + const filePath = resolve(cookbookDirectory(rootDir), slug, "goal.md"); + if (!existsSync(filePath)) return undefined; + return readFileSync(filePath, "utf-8"); +} + +/** Reads all present section files; throws when neither goal.md nor content.md exists. */ export function readContentSections( rootDir: string, section: FolderContentSection, slug: string, ): ContentSections { + const goal = readContentSection(rootDir, section, slug, "goal"); const content = readContentSection(rootDir, section, slug, "content"); - if (content === undefined) { + if (goal === undefined && content === undefined) { throw new Error( - `Missing required content.md for ${section} "${slug}" at content/${section}/${slug}/content.md`, + `Missing required goal.md or content.md for ${section} "${slug}" at content/${section}/${slug}/`, ); } const prerequisites = readContentSection( @@ -116,7 +132,9 @@ export function readContentSections( "prerequisites", ); const deployment = readContentSection(rootDir, section, slug, "deployment"); - const sections: ContentSections = { content }; + const sections: ContentSections = {}; + if (goal !== undefined) sections.goal = goal; + if (content !== undefined) sections.content = content; if (prerequisites !== undefined) sections.prerequisites = prerequisites; if (deployment !== undefined) sections.deployment = deployment; return sections; diff --git a/src/lib/content-sections.ts b/src/lib/content-sections.ts index 0269335..0de79d9 100644 --- a/src/lib/content-sections.ts +++ b/src/lib/content-sections.ts @@ -1,5 +1,6 @@ /** Allowed file names inside each content/// folder. */ const CONTENT_SECTION_FILES = [ + "goal", "content", "prerequisites", "deployment", @@ -10,11 +11,23 @@ export type ContentSectionFile = (typeof CONTENT_SECTION_FILES)[number]; export const REQUIRED_CONTENT_SECTION_FILE: ContentSectionFile = "content"; export type ContentSections = { - content: string; + goal?: string; + /** Present when content.md exists. Optional because examples may only have goal.md. */ + content?: string; prerequisites?: string; deployment?: string; }; +/** + * Returns goal-only content for agent prompts. Falls back to + * joinContentSections() when no goal.md exists (backward compat + * during incremental migration). + */ +export function goalOnly(sections: ContentSections): string { + if (sections.goal) return sections.goal.trim(); + return joinContentSections(sections); +} + /** Joins present sections in display order (prerequisites → content → deployment). */ export function joinContentSections(sections: ContentSections): string { const parts = [ @@ -24,3 +37,14 @@ export function joinContentSections(sections: ContentSections): string { ].filter((part): part is string => Boolean(part && part.trim())); return parts.map((part) => part.trim()).join("\n\n"); } + +/** Joins goal + content for human pages. Falls back to joinContentSections(). */ +export function joinGoalAndContent(sections: ContentSections): string { + const parts = [ + sections.goal, + sections.prerequisites, + sections.content, + sections.deployment, + ].filter((part): part is string => Boolean(part && part.trim())); + return parts.map((part) => part.trim()).join("\n\n"); +} diff --git a/src/lib/cookbook-composition.ts b/src/lib/cookbook-composition.ts index 04bcbbd..35c02c8 100644 --- a/src/lib/cookbook-composition.ts +++ b/src/lib/cookbook-composition.ts @@ -1,4 +1,4 @@ -import type { ContentSections } from "@/lib/content-sections"; +import { goalOnly, type ContentSections } from "@/lib/content-sections"; export type CookbookRecipeInput = { id: string; @@ -6,11 +6,15 @@ export type CookbookRecipeInput = { sections: ContentSections; }; +export type CookbookCompositionMode = "agent" | "human"; + type CookbookCompositionInput = { cookbookName: string; cookbookDescription: string; + goal?: string; intro?: string; recipes: CookbookRecipeInput[]; + mode?: CookbookCompositionMode; }; /** Strips a leading `## Prerequisites` heading (and any blank line that follows) from a prereqs body. */ @@ -34,18 +38,36 @@ function wrapRecipeDeployment(recipe: CookbookRecipeInput): string | undefined { } /** - * Reshuffles a cookbook into: intro → combined Prerequisites → all recipe content bodies → - * optional combined Deployment. Recipe content.md bodies keep their own `## ` title - * so they land as peer sections; prereqs are demoted to H3 under one shared H2. + * Composes a cookbook from its constituent recipes. + * + * mode="human" (default): intro/goal → combined Prerequisites → all recipe + * content bodies → optional combined Deployment. + * + * mode="agent": cookbook goal/intro → each recipe's goal under a + * "## Component: " heading. No prerequisites, content bodies, or deployment. */ export function composeCookbookMarkdown( input: CookbookCompositionInput, ): string { - const { intro, recipes } = input; + const mode = input.mode ?? "human"; + const introText = input.goal ?? input.intro; + const { recipes } = input; const parts: string[] = []; - if (intro && intro.trim()) { - parts.push(intro.trim()); + if (introText && introText.trim()) { + parts.push(introText.trim()); + } + + if (mode === "agent") { + for (const recipe of recipes) { + const recipeGoal = goalOnly(recipe.sections); + if (recipeGoal.trim()) { + parts.push( + `${heading(2, `Component: ${recipe.name}`)}\n\n${recipeGoal.trim()}`, + ); + } + } + return parts.join("\n\n"); } const prereqBlocks = recipes @@ -56,8 +78,8 @@ export function composeCookbookMarkdown( } const contentBlocks = recipes - .map((recipe) => recipe.sections.content.trim()) - .filter((block) => Boolean(block)); + .map((recipe) => recipe.sections.content?.trim()) + .filter((block): block is string => Boolean(block)); if (contentBlocks.length > 0) { parts.push(contentBlocks.join("\n\n---\n\n")); } diff --git a/src/lib/copy-preamble.ts b/src/lib/copy-preamble.ts index 10ad049..3df28ad 100644 --- a/src/lib/copy-preamble.ts +++ b/src/lib/copy-preamble.ts @@ -111,7 +111,7 @@ function buildLocalBootstrapBlock( return [ "# Verify your local Databricks dev environment", "", - "A working Databricks CLI profile is the prerequisite for every step that follows. Walk the user through the recipe below — _even if they say their environment is already set up_. The verification steps are quick and prevent confusing failures further down.", + "A working Databricks CLI profile is the prerequisite for every step that follows. The goal below describes what a ready environment looks like. Use your installed Databricks agent skills to verify and set up the environment — _even if the user says their environment is already set up_.", "", absolutizeMarkdown(localBootstrap.trim(), siteOrigin), ].join("\n"); @@ -127,7 +127,7 @@ function buildTemplateBlock(kind: AgentPromptKind, body: string): string { return [ `# The ${label} the user copied`, "", - `The full ${label} prompt is below. This is what the user wants to focus on today. Once the local-bootstrap above passes and the intent questions are answered, work through this content step by step.`, + `The ${label} goal is below — it describes what the user wants to build. Once the local-bootstrap above passes and the intent questions are answered, use your installed Databricks agent skills to implement it.`, "", body.trim(), ].join("\n"); diff --git a/src/pages/templates/ai-chat-app.tsx b/src/pages/templates/ai-chat-app.tsx index ceab9b9..5106990 100644 --- a/src/pages/templates/ai-chat-app.tsx +++ b/src/pages/templates/ai-chat-app.tsx @@ -1,7 +1,7 @@ import type { ReactNode } from "react"; import { CookbookDetail } from "@/components/cookbooks/cookbook-detail"; import { useCookbookMarkdown } from "@/lib/use-cookbook-markdown"; -import Intro from "@site/content/cookbooks/ai-chat-app/intro.md"; +import Intro from "@site/content/cookbooks/ai-chat-app/goal.md"; import FoundationModelsApiPrereqs from "@site/content/recipes/foundation-models-api/prerequisites.md"; import FoundationModelsApiContent from "@site/content/recipes/foundation-models-api/content.md"; import AiChatModelServingPrereqs from "@site/content/recipes/ai-chat-model-serving/prerequisites.md"; diff --git a/tests/bootstrap-prompt.test.ts b/tests/bootstrap-prompt.test.ts index 1b535d2..8461ff5 100644 --- a/tests/bootstrap-prompt.test.ts +++ b/tests/bootstrap-prompt.test.ts @@ -115,13 +115,14 @@ describe("hero bootstrap prompt composition (matches /api/bootstrap-prompt)", () expect(combined).not.toContain("# The example the user copied"); }); - test("composed hero prompt includes recipe content (databricks -v) and llms.txt URL", () => { + test("composed hero prompt includes goal content and llms.txt URL", () => { const combined = composeAgentPrompt({ parts: loadAgentPromptParts(), kind: "hero", siteOrigin: "https://dev.databricks.com", }); - expect(combined).toContain("databricks -v"); + // goal.md has the outcome description, not CLI commands + expect(combined).toContain("Set Up Your Local Dev Environment"); expect(combined).toContain("https://dev.databricks.com/llms.txt"); }); @@ -130,7 +131,8 @@ describe("hero bootstrap prompt composition (matches /api/bootstrap-prompt)", () "recipes", "set-up-your-local-dev-environment", ); - expect(recipe).toContain("## Set Up Your Local Dev Environment"); - expect(recipe).toContain("databricks -v"); + expect(recipe).toContain("Set Up Your Local Dev Environment"); + // Agent prompt now returns goal.md content (outcome), not full implementation + expect(recipe).toContain("authenticated CLI profile"); }); }); diff --git a/tests/markdown.test.ts b/tests/markdown.test.ts index 5901019..c030667 100644 --- a/tests/markdown.test.ts +++ b/tests/markdown.test.ts @@ -60,13 +60,14 @@ describe("detail markdown resolver", () => { expect(frontmatterCount).toBe(2); }); - test("resolves recipe markdown", () => { + test("resolves recipe markdown (returns goal when goal.md exists)", () => { const markdown = getDetailMarkdown( "recipes", "set-up-your-local-dev-environment", ); - expect(markdown).toContain("## Set Up Your Local Dev Environment"); - expect(markdown).toContain("databricks -v"); + expect(markdown).toContain("Set Up Your Local Dev Environment"); + // Agent prompt now returns goal.md content, not full implementation + expect(markdown).toContain("authenticated CLI profile"); }); test("resolves example markdown", () => { @@ -75,10 +76,11 @@ describe("detail markdown resolver", () => { expect(markdown).toContain("Data Flow"); }); - test("resolves template markdown", () => { + test("resolves template markdown (cookbook in agent mode)", () => { const markdown = getDetailMarkdown("templates", "ai-chat-app"); expect(markdown).toContain("# AI Chat App"); - expect(markdown).toContain("## Lakebase Agent Memory"); + // Agent mode uses "Component:" headings + expect(markdown).toContain("Component: Lakebase Agent Memory"); }); test("template markdown no longer embeds the local dev environment recipe (now injected by the meta-prompt)", () => { @@ -87,33 +89,21 @@ describe("detail markdown resolver", () => { expect(markdown).not.toMatch(/^### Set Up Your Local Dev Environment$/m); }); - test("template markdown hoists all recipe prereqs before any recipe content", () => { + test("template markdown for cookbook uses agent mode (goals only, no prereqs or full content)", () => { const markdown = getDetailMarkdown("templates", "ai-chat-app"); - - const firstLineStart = (pattern: RegExp): number => - markdown.search(pattern); - - const prereqIdx = firstLineStart(/^## Prerequisites$/m); - const foundationContentIdx = firstLineStart( - /^## Query AI Gateway Endpoints$/m, - ); - const lakebaseContentIdx = firstLineStart(/^## Lakebase Agent Memory$/m); - - expect(prereqIdx).toBeGreaterThanOrEqual(0); - expect(prereqIdx).toBeLessThan(foundationContentIdx); - expect(foundationContentIdx).toBeLessThan(lakebaseContentIdx); - // Only one combined `## Prerequisites` heading, with demoted H3 per recipe. - expect(markdown.match(/^## Prerequisites$/gm)?.length).toBe(1); - expect(markdown).toMatch(/^### Query AI Gateway Endpoints$/m); - expect(markdown).toMatch(/^### Lakebase Agent Memory$/m); + // Agent mode: cookbook goal + recipe goals as components + expect(markdown).toContain("What you are building"); + expect(markdown).toContain("## Component:"); + // Agent mode should NOT include prerequisites section + expect(markdown).not.toContain("## Prerequisites"); }); - test("template markdown includes cookbook intro.md above Prerequisites when present", () => { + test("template markdown includes cookbook goal.md above recipe goals", () => { const markdown = getDetailMarkdown("templates", "ai-chat-app"); - const introIdx = markdown.indexOf("## What you are building"); - const prereqIdx = markdown.indexOf("## Prerequisites"); - expect(introIdx).toBeGreaterThanOrEqual(0); - expect(introIdx).toBeLessThan(prereqIdx); + const goalIdx = markdown.indexOf("## What you are building"); + const componentIdx = markdown.indexOf("## Component:"); + expect(goalIdx).toBeGreaterThanOrEqual(0); + expect(goalIdx).toBeLessThan(componentIdx); expect(markdown).toContain("How the steps fit together"); }); @@ -141,7 +131,7 @@ describe("templates section resolves recipes, examples, and cookbooks", () => { test("resolves a cookbook slug via templates", () => { const markdown = getDetailMarkdown("templates", "ai-chat-app"); expect(markdown).toContain("# AI Chat App"); - expect(markdown).toContain("## Lakebase Agent Memory"); + expect(markdown).toContain("Component: Lakebase Agent Memory"); }); test("throws for unknown template slug", () => { diff --git a/tests/validate-content.test.ts b/tests/validate-content.test.ts index 172648d..6ae3109 100644 --- a/tests/validate-content.test.ts +++ b/tests/validate-content.test.ts @@ -96,7 +96,7 @@ describe("validate-content script", () => { const result = runValidator(workDir); expect(result.status).toBe(1); expect(result.stderr).toContain("no-content"); - expect(result.stderr).toContain("missing the required content.md"); + expect(result.stderr).toContain("missing a required file"); }); test("fails when a folder contains a disallowed filename", () => { From 19d358659e67dab39cd15eef07b4df76d37b0ac8 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Mon, 18 May 2026 21:41:49 +0200 Subject: [PATCH 2/6] chore: remove brainstorm doc from PR Co-authored-by: Isaac --- .../examples/agentic-support-console/goal.md | 2 - content/examples/content-moderator/goal.md | 2 - .../examples/inventory-intelligence/goal.md | 2 - content/examples/rag-chat/goal.md | 2 - content/examples/saas-tracker/goal.md | 2 - content/examples/vacation-rentals/goal.md | 2 - content/recipes/ai-chat-model-serving/goal.md | 2 - content/recipes/embeddings-generation/goal.md | 2 - content/recipes/foundation-models-api/goal.md | 2 - .../genie-conversational-analytics/goal.md | 2 - content/recipes/genie-multi-space/goal.md | 2 - content/recipes/lakebase-agent-memory/goal.md | 2 - .../goal.md | 2 - .../recipes/lakebase-create-instance/goal.md | 2 - .../recipes/lakebase-data-persistence/goal.md | 2 - .../lakebase-drizzle-off-platform/goal.md | 2 - .../goal.md | 2 - content/recipes/lakebase-pgvector/goal.md | 2 - .../recipes/lakebase-token-management/goal.md | 2 - .../medallion-architecture-from-cdc/goal.md | 2 - .../model-serving-endpoint-creation/goal.md | 2 - .../recipes/onboard-your-coding-agent/goal.md | 2 - .../set-up-your-local-dev-environment/goal.md | 2 - .../recipes/spin-up-databricks-app/goal.md | 2 - .../recipes/sync-tables-autoscaling/goal.md | 2 - content/recipes/unity-catalog-setup/goal.md | 2 - content/recipes/volume-file-upload/goal.md | 2 - ...8-agent-skills-unification-requirements.md | 130 ------------------ tests/api-markdown.test.ts | 2 +- tests/bootstrap-prompt.test.ts | 6 +- tests/markdown.test.ts | 14 +- 31 files changed, 11 insertions(+), 195 deletions(-) delete mode 100644 docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md diff --git a/content/examples/agentic-support-console/goal.md b/content/examples/agentic-support-console/goal.md index fbe804a..7788369 100644 --- a/content/examples/agentic-support-console/goal.md +++ b/content/examples/agentic-support-console/goal.md @@ -1,5 +1,3 @@ -## Agentic Support Console - This template brings together the full Databricks developer stack into a single operational data application: an AI-powered support console where every customer message is automatically triaged by an LLM, and support agents review, approve, or override the suggestion from a purpose-built internal tool. ### Data Flow diff --git a/content/examples/content-moderator/goal.md b/content/examples/content-moderator/goal.md index 36eedad..133cab1 100644 --- a/content/examples/content-moderator/goal.md +++ b/content/examples/content-moderator/goal.md @@ -1,5 +1,3 @@ -## Content Moderator - This template demonstrates an internal content moderation tool built on Databricks: authors submit content for different channels (company blog, LinkedIn, Twitter, newsletter, press releases), moderators maintain per-channel guidelines, and an LLM scores each submission against those guidelines before a human reviewer makes the final call. ### Data Flow diff --git a/content/examples/inventory-intelligence/goal.md b/content/examples/inventory-intelligence/goal.md index e95119f..c261903 100644 --- a/content/examples/inventory-intelligence/goal.md +++ b/content/examples/inventory-intelligence/goal.md @@ -1,5 +1,3 @@ -## Inventory Intelligence - This template builds a full retail inventory management system on the Databricks stack: a React app where store managers monitor stock health, review AI-generated replenishment recommendations, and approve purchase orders — all powered by a live medallion pipeline and pluggable demand forecast job. ### Data Flow diff --git a/content/examples/rag-chat/goal.md b/content/examples/rag-chat/goal.md index e54cf79..88fb0e9 100644 --- a/content/examples/rag-chat/goal.md +++ b/content/examples/rag-chat/goal.md @@ -1,5 +1,3 @@ -## RAG Chat App - This template demonstrates a Retrieval-Augmented Generation chat app built on Databricks: a user question is embedded, similar documents are retrieved from a pgvector store in Lakebase Postgres, and the retrieved context is injected into a Model Serving call that streams the answer back. Conversations and sources are persisted per chat in Lakebase. ### Data Flow diff --git a/content/examples/saas-tracker/goal.md b/content/examples/saas-tracker/goal.md index 9a57829..8451358 100644 --- a/content/examples/saas-tracker/goal.md +++ b/content/examples/saas-tracker/goal.md @@ -1,5 +1,3 @@ -## SaaS Subscription Tracker - This template demonstrates a straightforward internal CRUD tool built on Databricks: a SaaS subscription tracker where teams log the tools they use, who owns each subscription, what it costs, and when it renews. A Genie space provides self-serve analytics over the subscription data. ### Data Flow diff --git a/content/examples/vacation-rentals/goal.md b/content/examples/vacation-rentals/goal.md index 9e045a3..c91bb73 100644 --- a/content/examples/vacation-rentals/goal.md +++ b/content/examples/vacation-rentals/goal.md @@ -1,5 +1,3 @@ -## Vacation Rentals Operations Console - This template demonstrates an internal operations console for a vacation rentals platform ("Wanderbricks"). Operators see revenue performance by destination, work through a booking queue with per-booking flags and agent notes, and ask natural-language questions about the business through an embedded Genie chat panel. ### Data Flow diff --git a/content/recipes/ai-chat-model-serving/goal.md b/content/recipes/ai-chat-model-serving/goal.md index 40ccc98..e652b40 100644 --- a/content/recipes/ai-chat-model-serving/goal.md +++ b/content/recipes/ai-chat-model-serving/goal.md @@ -1,5 +1,3 @@ -## Streaming AI Chat with Model Serving - Build a streaming AI chat experience in a Databricks App using Databricks Model Serving with OpenAI-compatible endpoints. When done, you will have: diff --git a/content/recipes/embeddings-generation/goal.md b/content/recipes/embeddings-generation/goal.md index 6a21689..f59d640 100644 --- a/content/recipes/embeddings-generation/goal.md +++ b/content/recipes/embeddings-generation/goal.md @@ -1,5 +1,3 @@ -## Generate Embeddings with AI Gateway - Generate text embeddings from a Databricks AI Gateway endpoint using the Databricks SDK. When done, you will have: diff --git a/content/recipes/foundation-models-api/goal.md b/content/recipes/foundation-models-api/goal.md index c014b93..0068164 100644 --- a/content/recipes/foundation-models-api/goal.md +++ b/content/recipes/foundation-models-api/goal.md @@ -1,5 +1,3 @@ -## Query AI Gateway Endpoints - Access Databricks foundation models through AI Gateway endpoints with built-in governance, monitoring, and streaming support. When done, you will have: diff --git a/content/recipes/genie-conversational-analytics/goal.md b/content/recipes/genie-conversational-analytics/goal.md index eac2146..2ea9039 100644 --- a/content/recipes/genie-conversational-analytics/goal.md +++ b/content/recipes/genie-conversational-analytics/goal.md @@ -1,5 +1,3 @@ -## Genie Conversational Analytics - Embed a Databricks AI/BI Genie chat interface in your app so users can explore data through natural language. When done, you will have: diff --git a/content/recipes/genie-multi-space/goal.md b/content/recipes/genie-multi-space/goal.md index e236418..f3efa3c 100644 --- a/content/recipes/genie-multi-space/goal.md +++ b/content/recipes/genie-multi-space/goal.md @@ -1,5 +1,3 @@ -## Genie Multi-Space Selector - Upgrade a single-space Genie app to let users switch between multiple AI/BI Genie spaces from a dropdown. When done, you will have: diff --git a/content/recipes/lakebase-agent-memory/goal.md b/content/recipes/lakebase-agent-memory/goal.md index 382f76f..e4ab43f 100644 --- a/content/recipes/lakebase-agent-memory/goal.md +++ b/content/recipes/lakebase-agent-memory/goal.md @@ -1,5 +1,3 @@ -## Lakebase Agent Memory - Persist AI agent chat conversations to Lakebase so users can resume sessions, view full message history, and let the agent reason over previous turns across requests and deploys. When done, you will have: diff --git a/content/recipes/lakebase-change-data-feed-autoscaling/goal.md b/content/recipes/lakebase-change-data-feed-autoscaling/goal.md index e66000f..66b471e 100644 --- a/content/recipes/lakebase-change-data-feed-autoscaling/goal.md +++ b/content/recipes/lakebase-change-data-feed-autoscaling/goal.md @@ -1,5 +1,3 @@ -## Replicate Lakebase Tables to Unity Catalog with Lakehouse Sync - Replicate Lakebase Autoscaling Postgres tables into Unity Catalog as managed Delta tables using Lakehouse Sync, capturing every row-level change as SCD Type 2 history. When done, you will have: diff --git a/content/recipes/lakebase-create-instance/goal.md b/content/recipes/lakebase-create-instance/goal.md index c4f6b2f..23701be 100644 --- a/content/recipes/lakebase-create-instance/goal.md +++ b/content/recipes/lakebase-create-instance/goal.md @@ -1,5 +1,3 @@ -## Create a Lakebase Instance - Provision a managed Lakebase Postgres project on Databricks and collect the connection values needed by downstream recipes. When done, you will have: diff --git a/content/recipes/lakebase-data-persistence/goal.md b/content/recipes/lakebase-data-persistence/goal.md index 11c5d54..42cfe94 100644 --- a/content/recipes/lakebase-data-persistence/goal.md +++ b/content/recipes/lakebase-data-persistence/goal.md @@ -1,5 +1,3 @@ -## Lakebase Data Persistence - Add a managed Postgres database to your Databricks App using the Lakebase plugin, with schema setup, table creation, and full CRUD REST API routes. When done, you will have: diff --git a/content/recipes/lakebase-drizzle-off-platform/goal.md b/content/recipes/lakebase-drizzle-off-platform/goal.md index da77798..97c6ef0 100644 --- a/content/recipes/lakebase-drizzle-off-platform/goal.md +++ b/content/recipes/lakebase-drizzle-off-platform/goal.md @@ -1,5 +1,3 @@ -## Drizzle ORM with Lakebase in an Off-Platform App - Connect Drizzle ORM to Lakebase in any Node.js server outside Databricks App Platform, with automatic credential refresh and migration support. When done, you will have: diff --git a/content/recipes/lakebase-off-platform-env-management/goal.md b/content/recipes/lakebase-off-platform-env-management/goal.md index cb321c5..a88f62c 100644 --- a/content/recipes/lakebase-off-platform-env-management/goal.md +++ b/content/recipes/lakebase-off-platform-env-management/goal.md @@ -1,5 +1,3 @@ -## Lakebase Environment Management for Off-Platform Apps - Define and validate the environment variables needed to connect to Lakebase from apps deployed outside Databricks App Platform. When done, you will have: diff --git a/content/recipes/lakebase-pgvector/goal.md b/content/recipes/lakebase-pgvector/goal.md index 6906cd5..13f7507 100644 --- a/content/recipes/lakebase-pgvector/goal.md +++ b/content/recipes/lakebase-pgvector/goal.md @@ -1,5 +1,3 @@ -## Vector Search with Lakebase and pgvector - Enable vector similarity search in your Lakebase Postgres database using the pgvector extension, with a server-side module for storing and querying embeddings. When done, you will have: diff --git a/content/recipes/lakebase-token-management/goal.md b/content/recipes/lakebase-token-management/goal.md index 1aa0fc2..1fa3aba 100644 --- a/content/recipes/lakebase-token-management/goal.md +++ b/content/recipes/lakebase-token-management/goal.md @@ -1,5 +1,3 @@ -## Lakebase Token Management - Fetch, cache, and automatically refresh the short-lived Postgres credentials that Lakebase requires, supporting both token auth and M2M OAuth. When done, you will have: diff --git a/content/recipes/medallion-architecture-from-cdc/goal.md b/content/recipes/medallion-architecture-from-cdc/goal.md index a4e58b0..9480d73 100644 --- a/content/recipes/medallion-architecture-from-cdc/goal.md +++ b/content/recipes/medallion-architecture-from-cdc/goal.md @@ -1,5 +1,3 @@ -## Medallion Architecture from CDC History Tables - Transform Lakehouse Sync CDC history tables into a layered medallion architecture with bronze, silver, and gold layers using Lakeflow Declarative Pipelines. When done, you will have: diff --git a/content/recipes/model-serving-endpoint-creation/goal.md b/content/recipes/model-serving-endpoint-creation/goal.md index 463a5e9..3bb0135 100644 --- a/content/recipes/model-serving-endpoint-creation/goal.md +++ b/content/recipes/model-serving-endpoint-creation/goal.md @@ -1,5 +1,3 @@ -## Create a Databricks Model Serving Endpoint - Provision and validate a Databricks Model Serving endpoint for AI chat inference. When done, you will have: diff --git a/content/recipes/onboard-your-coding-agent/goal.md b/content/recipes/onboard-your-coding-agent/goal.md index b835092..f6c062a 100644 --- a/content/recipes/onboard-your-coding-agent/goal.md +++ b/content/recipes/onboard-your-coding-agent/goal.md @@ -1,5 +1,3 @@ -## Onboard Your Coding Agent - Make a Databricks repo agent-ready so your coding agent understands the Databricks platform and can fetch DevHub documentation on demand. When done, you will have: diff --git a/content/recipes/set-up-your-local-dev-environment/goal.md b/content/recipes/set-up-your-local-dev-environment/goal.md index bf2ede7..8d847c5 100644 --- a/content/recipes/set-up-your-local-dev-environment/goal.md +++ b/content/recipes/set-up-your-local-dev-environment/goal.md @@ -1,5 +1,3 @@ -## Set Up Your Local Dev Environment - Install the Databricks CLI, authenticate a profile, and verify the handshake. Every other DevHub template assumes this has already passed. After this step you will have: diff --git a/content/recipes/spin-up-databricks-app/goal.md b/content/recipes/spin-up-databricks-app/goal.md index dedd623..bf690d1 100644 --- a/content/recipes/spin-up-databricks-app/goal.md +++ b/content/recipes/spin-up-databricks-app/goal.md @@ -1,5 +1,3 @@ -## Spin Up a Databricks App - Generate a working AppKit Databricks App from scratch and deploy it to your workspace. When done, you will have: diff --git a/content/recipes/sync-tables-autoscaling/goal.md b/content/recipes/sync-tables-autoscaling/goal.md index a83bd28..89ec199 100644 --- a/content/recipes/sync-tables-autoscaling/goal.md +++ b/content/recipes/sync-tables-autoscaling/goal.md @@ -1,5 +1,3 @@ -## Sync a Unity Catalog Table to Lakebase - Serve lakehouse data through Lakebase Autoscaling Postgres so your applications can query it with sub-10ms latency using a synced table that stays up to date automatically. When done, you will have: diff --git a/content/recipes/unity-catalog-setup/goal.md b/content/recipes/unity-catalog-setup/goal.md index c1c23d3..e946594 100644 --- a/content/recipes/unity-catalog-setup/goal.md +++ b/content/recipes/unity-catalog-setup/goal.md @@ -1,5 +1,3 @@ -## Set Up Unity Catalog with External Storage - Create a Unity Catalog catalog backed by an external S3 bucket for scenarios that require custom storage control. When done, you will have: diff --git a/content/recipes/volume-file-upload/goal.md b/content/recipes/volume-file-upload/goal.md index 9fe9161..b340426 100644 --- a/content/recipes/volume-file-upload/goal.md +++ b/content/recipes/volume-file-upload/goal.md @@ -1,5 +1,3 @@ -## Volume File Manager - Add file upload, browsing, download, delete, file type validation, and CSV row preview to your Databricks App using Unity Catalog Volumes. When done, you will have: diff --git a/docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md b/docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md deleted file mode 100644 index cfb4544..0000000 --- a/docs/brainstorms/2026-05-18-agent-skills-unification-requirements.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -date: 2026-05-18 -topic: agent-skills-unification ---- - -# Unifying Agent Skills with DevHub Templates - -## Problem Frame - -DevHub templates (recipes, cookbooks, examples) and agent skills both contain implementation guidance -- CLI commands, code patterns, configuration. The content overlaps significantly, causing: - -1. **Contradictions** -- recipe says CLI 0.296+, skill says 0.292+. Agents receiving both must reconcile. -2. **Drift** -- when one side updates, the other gets stale. No sync mechanism exists. -3. **Bloated agent prompts** -- agents receive full step-by-step recipe content AND skills covering the same topic. Redundant tokens, conflicting instructions. - -PR #95 partially addressed this by adding "skills = source of truth" directives to intent blocks and converting examples to outcome-only. But recipes still send full implementation content to agents, creating the contradiction window. - -## Key Decisions - -- **Skills are the source of truth** for implementation (CLI commands, code patterns, configuration). Recipe content is authoritative for human-facing tutorials but not for agent prompts. -- **Agent-first does not mean human-hostile.** Humans keep full step-by-step instructions on the web page. The change is what agents receive, not what humans see. -- **Physical file boundary** separates agent content from human content. No section markers or parsing -- different files for different audiences. - -## Requirements - -**File Structure** - -- R1. All template types (recipes, cookbooks, examples) use `goal.md` for the outcome description. This file describes WHAT you'll build, key decisions, services involved, and what the result looks like. -- R2. Recipes have a second file `content.md` containing human-targeted step-by-step instructions. This includes prerequisites (folded in from current `prerequisites.md`) and implementation steps. -- R3. Cookbooks have only `goal.md` (replaces the current `intro.md`). It describes the overall composed app and how the pieces fit together. -- R4. Examples have only `goal.md` (renamed from current `content.md`). Examples are already outcome-only. -- R5. Delete all `prerequisites.md` files. Their content moves into the top of each recipe's `content.md`. - -**Agent Prompt Composition** - -- R6. Agent prompt for recipes includes only `goal.md` (not `content.md`). Skills handle implementation. -- R7. Agent prompt for cookbooks includes the cookbook's `goal.md` followed by each constituent recipe's `goal.md`. No recipe `content.md` is sent. -- R8. Agent prompt for examples includes only `goal.md` (no behavioral change since examples are already outcome-only, just reads from the renamed file). -- R9. The local bootstrap recipe (`set-up-your-local-dev-environment`) agent prompt also sends only `goal.md`. Skills and the dev-guidelines preamble handle environment verification. - -**Human Page Rendering** - -- R10. Recipe detail pages render `goal.md` + `content.md` joined together. The outcome description appears at the top, followed by the full step-by-step instructions. Visual separation uses existing `prose-solution` CSS heading borders (h2 gets `border-b` automatically) -- no new components needed. -- R11. Cookbook detail pages keep inline composition: cookbook `goal.md` at top, then each recipe's `goal.md` + `content.md` inlined (same reading experience as today). -- R12. Example detail pages render `goal.md` (no behavioral change, just reads from the renamed file). - -**Cookbook Composition** - -- R13. `composeCookbookMarkdown()` gains an optional mode parameter (defaulting to current behavior) to compose agent-only output (goals only) vs human output (goals + content). Single function with mode parameter preferred over two separate functions to avoid duplicating ~40 lines of shared composition logic. -- R14. All 5 cookbooks get a `goal.md`. The existing `ai-chat-app/intro.md` becomes `ai-chat-app/goal.md`. The other 4 cookbooks need new `goal.md` files written. -- R20. Cookbook `goal.md` explicitly frames the overall outcome as composed of multiple components and lists the constituent recipe goals. The agent prompt composition function wraps each recipe's `goal.md` with a labeled heading (e.g., "Component: Set Up Model Serving") so agents understand the hierarchy -- big goal at the top, sub-goals below. - -**Content Migration** - -- R15. For each of the 21 recipes: extract the outcome description from current `content.md` into a new `goal.md`, leave the implementation in `content.md`, fold `prerequisites.md` content into the top of `content.md`. Effort breakdown: 8 recipes have clear outcome intros (easy extraction), 10 dive into steps (need 2-3 sentence outcomes written), 3 need significant writing (Lakehouse Sync CDC, Token Management, Sync Tables). -- R16. For each of the 6 examples: rename `content.md` to `goal.md`. -- R17. The `rag-chat` example's `deployment.md` is implementation-oriented (npm commands, env hydration, workspace ID injection). It should NOT fold into `goal.md`. Since the example points to a GitHub repo and `template/README.md` is the deployment runbook, delete `deployment.md` (it's redundant with the repo README). - -**Intent Blocks and Preamble** - -- R18. Update `content/intent-recipe.md` to reflect that the agent receives only the goal, not implementation steps. Remove references to "follow the recipe step by step" for the implementation path; instead direct agents to use installed skills. -- R19. Update `content/intent-cookbook.md` similarly -- the agent receives goals for each recipe, not full content. - -## Flow Diagram - -``` - Recipe folder - / \ - goal.md content.md - (outcome) (human instructions) - | | - +-------+-------+ | - | | | - Agent prompt Human page Human page - (goal only) (goal + content joined) - | - v - Installed skills - (implementation) -``` - -``` - Cookbook folder - | - goal.md - (overall outcome) - | - +--------+--------+ - | | - Agent prompt Human page - (cookbook goal + (cookbook goal + - recipe goals) recipe goals + - | recipe content - v inlined) - Installed skills -``` - -## Success Criteria - -- Agent prompts for recipes contain zero implementation details (no CLI commands, no code blocks, no version requirements). Only outcome descriptions and key decisions. -- Human recipe pages render identically to today's experience (same content, same structure, same reading flow) except the outcome section appears prominently at the top. -- No contradictions possible between recipe agent content and skills -- they cover different concerns (scope vs implementation). -- Cookbook agent prompts are significantly shorter (goal descriptions only, not full recipe bodies). - -## Scope Boundaries - -- **Not changing recipe content for humans.** The step-by-step instructions stay on the web page. We're changing what agents receive, not what humans read. -- **Not syncing content between repos.** Skills and recipe content are maintained independently. They serve different audiences with different formats. -- **Not changing the agent-skills repo.** The incorporate-devhub-recipes branch is a separate workstream. This PR focuses on the DevHub side. -- **Not changing the example template/GitHub repo pattern.** Examples already work well. - -## Dependencies / Assumptions - -- Agent skills are installed in the user's environment (dev-guidelines already requires this). -- The incorporate-devhub-recipes branch in the agent-skills repo lands independently. It already contains the implementation content that agents need. -- The `goal.md` content for each recipe needs to be written well enough that agents understand the scope without implementation details. - -## Resolved Questions - -All questions originally deferred to planning have been investigated and resolved: - -- **Local bootstrap (R9):** `databricks-core` skill fully covers CLI install (all OS paths), OAuth auth, profile verification, and smoke testing. Actually more comprehensive than the recipe. No special treatment needed. -- **Cookbook composition (R13):** Mode parameter on existing function. Avoids duplicating ~40 lines of shared logic. Backward-compatible default. -- **Recipe outcome boundaries (R15):** 8 recipes have clear outcome intros (easy extraction), 10 dive into steps (need short outcomes written), 3 need significant writing. -- **rag-chat deployment.md (R17):** Implementation-oriented, not outcome-oriented. Redundant with `template/README.md` in the GitHub repo. Delete it. -- **Visual separation (R10):** Existing `prose-solution` CSS heading borders provide natural separation. No new components. - -## Next Steps - --> `/ce:plan` for structured implementation planning diff --git a/tests/api-markdown.test.ts b/tests/api-markdown.test.ts index 9b09d50..cbb29de 100644 --- a/tests/api-markdown.test.ts +++ b/tests/api-markdown.test.ts @@ -179,7 +179,7 @@ describe("/api/markdown about-devhub preamble policy", () => { }); expect(result.statusCode).toBe(200); expect(result.body.startsWith("# About DevHub")).toBe(true); - expect(result.body).toContain("## Agentic Support Console"); + expect(result.body).toContain("Agentic Support Console"); }); test("templates index does NOT include the preamble", () => { diff --git a/tests/bootstrap-prompt.test.ts b/tests/bootstrap-prompt.test.ts index 8461ff5..e5229d1 100644 --- a/tests/bootstrap-prompt.test.ts +++ b/tests/bootstrap-prompt.test.ts @@ -101,7 +101,7 @@ describe("hero bootstrap prompt composition (matches /api/bootstrap-prompt)", () "# Verify your local Databricks dev environment", ); const localBootstrapBodyIdx = combined.indexOf( - "## Set Up Your Local Dev Environment", + "Install the Databricks CLI", ); expect(aboutIdx).toBe(0); @@ -122,7 +122,7 @@ describe("hero bootstrap prompt composition (matches /api/bootstrap-prompt)", () siteOrigin: "https://dev.databricks.com", }); // goal.md has the outcome description, not CLI commands - expect(combined).toContain("Set Up Your Local Dev Environment"); + expect(combined).toContain("Install the Databricks CLI"); expect(combined).toContain("https://dev.databricks.com/llms.txt"); }); @@ -131,8 +131,8 @@ describe("hero bootstrap prompt composition (matches /api/bootstrap-prompt)", () "recipes", "set-up-your-local-dev-environment", ); - expect(recipe).toContain("Set Up Your Local Dev Environment"); // Agent prompt now returns goal.md content (outcome), not full implementation + expect(recipe).toContain("Install the Databricks CLI"); expect(recipe).toContain("authenticated CLI profile"); }); }); diff --git a/tests/markdown.test.ts b/tests/markdown.test.ts index c030667..15f3e1d 100644 --- a/tests/markdown.test.ts +++ b/tests/markdown.test.ts @@ -65,14 +65,14 @@ describe("detail markdown resolver", () => { "recipes", "set-up-your-local-dev-environment", ); - expect(markdown).toContain("Set Up Your Local Dev Environment"); - // Agent prompt now returns goal.md content, not full implementation + // Agent prompt returns goal.md content (no heading, just description) + expect(markdown).toContain("Install the Databricks CLI"); expect(markdown).toContain("authenticated CLI profile"); }); test("resolves example markdown", () => { const markdown = getDetailMarkdown("examples", "agentic-support-console"); - expect(markdown).toContain("## Agentic Support Console"); + expect(markdown).toContain("AI-powered support console"); expect(markdown).toContain("Data Flow"); }); @@ -120,12 +120,12 @@ describe("templates section resolves recipes, examples, and cookbooks", () => { "templates", "set-up-your-local-dev-environment", ); - expect(markdown).toContain("## Set Up Your Local Dev Environment"); + expect(markdown).toContain("Install the Databricks CLI"); }); test("resolves an example slug via templates", () => { const markdown = getDetailMarkdown("templates", "agentic-support-console"); - expect(markdown).toContain("## Agentic Support Console"); + expect(markdown).toContain("AI-powered support console"); }); test("resolves a cookbook slug via templates", () => { @@ -271,7 +271,7 @@ describe("slug normalization strips .md extension", () => { "recipes", "set-up-your-local-dev-environment.md", ); - expect(markdown).toContain("## Set Up Your Local Dev Environment"); + expect(markdown).toContain("Install the Databricks CLI"); }); test("templates slug with .md extension resolves", () => { @@ -279,6 +279,6 @@ describe("slug normalization strips .md extension", () => { "templates", "agentic-support-console.md", ); - expect(markdown).toContain("## Agentic Support Console"); + expect(markdown).toContain("AI-powered support console"); }); }); From c3dd4f4fe953ac181ca2e61b9cf170fb2dd39975 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Mon, 18 May 2026 22:28:27 +0200 Subject: [PATCH 3/6] fix(examples): drop prerequisites from agent prompt, use goal-only body buildFullPrompt() now uses sections.goal when present, skipping prerequisites/content/deployment. The agent gets the outcome description + scaffold entry point; skills handle implementation. Only rag-chat was affected (the sole example with prerequisites.md). Renamed its prerequisites.md to content.md so the human page still shows the Lakebase provisioning steps. Co-authored-by: Isaac --- .../rag-chat/{prerequisites.md => content.md} | 0 src/lib/examples/build-example-markdown.ts | 69 ++++++++++++------- 2 files changed, 43 insertions(+), 26 deletions(-) rename content/examples/rag-chat/{prerequisites.md => content.md} (100%) diff --git a/content/examples/rag-chat/prerequisites.md b/content/examples/rag-chat/content.md similarity index 100% rename from content/examples/rag-chat/prerequisites.md rename to content/examples/rag-chat/content.md diff --git a/src/lib/examples/build-example-markdown.ts b/src/lib/examples/build-example-markdown.ts index 8ddbbd6..370831d 100644 --- a/src/lib/examples/build-example-markdown.ts +++ b/src/lib/examples/build-example-markdown.ts @@ -80,39 +80,56 @@ export function buildFullPrompt( includedRecipes, baseUrl, } = opts; + const hasGoal = Boolean(sections.goal); const cliTemplateUrl = `https://github.com/databricks/devhub/tree/main/${example.githubPath}`; - const lines: string[] = [ - `# ${example.name}`, - "", - example.description, - "", - "## Get started", - "", - ]; + const lines: string[] = [`# ${example.name}`, "", example.description, ""]; + + // When goal.md exists, use it as the body and skip prerequisites/content/deployment. + // The agent gets the outcome description + scaffold entry point; skills handle implementation. + if (hasGoal) { + lines.push(sections.goal!.trim(), ""); + } + + lines.push("## Get started", ""); if (isInitCommand(example.initCommand)) { - const hasPrereqs = Boolean(sections.prerequisites); - const hasDeployBlock = Boolean(sections.deployment); + if (!hasGoal) { + const hasPrereqs = Boolean(sections.prerequisites); + const hasDeployBlock = Boolean(sections.deployment); - if (hasPrereqs) { - lines.push(sections.prerequisites!, ""); - } + if (hasPrereqs) { + lines.push(sections.prerequisites!, ""); + } - lines.push( - "### Scaffold the project", - "", - "Run the command below to scaffold this example into a new directory using the [AppKit template system](/docs/appkit/v0/development/templates). It creates the app in your workspace, binds required resources, and writes a local `.env` with connection details resolved by the AppKit plugins.", - "", - "```bash", - example.initCommand, - "```", - "", - ); + lines.push( + "### Scaffold the project", + "", + "Run the command below to scaffold this example into a new directory using the [AppKit template system](/docs/appkit/v0/development/templates). It creates the app in your workspace, binds required resources, and writes a local `.env` with connection details resolved by the AppKit plugins.", + "", + "```bash", + example.initCommand, + "```", + "", + ); - if (hasDeployBlock) { - lines.push(sections.deployment!, ""); + if (hasDeployBlock) { + lines.push(sections.deployment!, ""); + } else { + lines.push( + "A **`README.md`** ships inside the scaffolded project. Follow it end to end to configure, run, and deploy the app.", + "", + ); + } } else { lines.push( + "### Scaffold the project", + "", + "Run the command below to scaffold this example into a new directory using the [AppKit template system](/docs/appkit/v0/development/templates). It creates the app in your workspace, binds required resources, and writes a local `.env` with connection details resolved by the AppKit plugins.", + "", + "```bash", + example.initCommand, + "```", + "", "A **`README.md`** ships inside the scaffolded project. Follow it end to end to configure, run, and deploy the app.", "", ); @@ -138,7 +155,7 @@ export function buildFullPrompt( ); } - if (sections.content) { + if (!hasGoal && sections.content) { lines.push("", sections.content); } From 036f0f62587b5694a2b3b2d41865ba7d30ad2f7b Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Tue, 19 May 2026 10:17:30 +0200 Subject: [PATCH 4/6] fix: browser cookbook copy path, missing page imports, dead code - plugins/cookbooks.ts now reads goal.md (falls back to intro.md) - use-cookbook-markdown.ts passes mode: "agent" for copy prompt - All 5 cookbook pages import and render Goal component - Strip headings from cookbook goal.md files - Remove unused joinGoalAndContent() - Update stale REQUIRED_CONTENT_SECTION_FILE comment Co-authored-by: Isaac --- content/cookbooks/ai-chat-app/goal.md | 2 -- content/cookbooks/app-with-lakebase/goal.md | 2 -- content/cookbooks/genie-analytics-app/goal.md | 2 -- .../cookbooks/lakebase-off-platform/goal.md | 2 -- .../operational-data-analytics/goal.md | 2 -- plugins/cookbooks.ts | 18 ++++++++++++++---- src/lib/content-sections.ts | 16 ++++------------ src/lib/use-cookbook-markdown.ts | 14 +++++++------- src/lib/use-raw-content-markdown.ts | 8 ++++++++ src/pages/templates/app-with-lakebase.tsx | 2 ++ src/pages/templates/genie-analytics-app.tsx | 2 ++ src/pages/templates/lakebase-off-platform.tsx | 2 ++ .../templates/operational-data-analytics.tsx | 2 ++ tests/markdown.test.ts | 4 ++-- 14 files changed, 43 insertions(+), 35 deletions(-) diff --git a/content/cookbooks/ai-chat-app/goal.md b/content/cookbooks/ai-chat-app/goal.md index 2c02796..7dd70d1 100644 --- a/content/cookbooks/ai-chat-app/goal.md +++ b/content/cookbooks/ai-chat-app/goal.md @@ -1,5 +1,3 @@ -## What you are building - A streaming AI chat app on Databricks: a user sends a message, the server authenticates with the Databricks CLI profile (or a service-principal token in production), calls an AI Gateway chat endpoint via the OpenAI-compatible provider, and streams the answer back token-by-token. Chat sessions and messages are persisted in Lakebase Postgres so conversations survive page refreshes and redeploys. ### How the steps fit together diff --git a/content/cookbooks/app-with-lakebase/goal.md b/content/cookbooks/app-with-lakebase/goal.md index 06d0adb..3bd7c8f 100644 --- a/content/cookbooks/app-with-lakebase/goal.md +++ b/content/cookbooks/app-with-lakebase/goal.md @@ -1,5 +1,3 @@ -## What you are building - A Databricks App with Lakebase Postgres for persistent data storage. The app has schema setup, full CRUD API routes, and deploys to the Databricks Apps platform. ### Components diff --git a/content/cookbooks/genie-analytics-app/goal.md b/content/cookbooks/genie-analytics-app/goal.md index ef90d00..8755fcb 100644 --- a/content/cookbooks/genie-analytics-app/goal.md +++ b/content/cookbooks/genie-analytics-app/goal.md @@ -1,5 +1,3 @@ -## What you are building - A minimal Databricks App with AI/BI Genie conversational analytics. Users ask natural-language questions about their data and get SQL-powered answers through an embedded Genie chat interface. ### Components diff --git a/content/cookbooks/lakebase-off-platform/goal.md b/content/cookbooks/lakebase-off-platform/goal.md index 6e0bffd..6cbdad1 100644 --- a/content/cookbooks/lakebase-off-platform/goal.md +++ b/content/cookbooks/lakebase-off-platform/goal.md @@ -1,5 +1,3 @@ -## What you are building - A connection from an app hosted outside the Databricks Apps platform (for example on AWS, Vercel, or Netlify) to Lakebase Postgres. The app uses portable environment configuration, token management with automatic credential refresh, and Drizzle ORM for type-safe database access. ### Components diff --git a/content/cookbooks/operational-data-analytics/goal.md b/content/cookbooks/operational-data-analytics/goal.md index 17cf873..371f320 100644 --- a/content/cookbooks/operational-data-analytics/goal.md +++ b/content/cookbooks/operational-data-analytics/goal.md @@ -1,5 +1,3 @@ -## What you are building - An end-to-end operational data analytics pipeline: data flows from an OLTP database (Lakebase Postgres) through CDC replication into Unity Catalog, gets transformed through a medallion architecture (bronze/silver/gold layers), and is ready for dashboards and downstream consumers. ### Components diff --git a/plugins/cookbooks.ts b/plugins/cookbooks.ts index 239a95e..9258135 100644 --- a/plugins/cookbooks.ts +++ b/plugins/cookbooks.ts @@ -1,12 +1,15 @@ import type { LoadContext, Plugin } from "@docusaurus/types"; import { getCookbookSlugs, + readCookbookGoal, readCookbookIntro, } from "../src/lib/content-markdown"; import { cookbooks } from "../src/lib/recipes/recipes"; type CookbooksGlobalData = { - /** Raw `content/cookbooks//intro.md` bodies keyed by cookbook id. */ + /** Raw `content/cookbooks//goal.md` bodies keyed by cookbook id. Falls back to intro.md. */ + goalsBySlug: Record; + /** @deprecated Use goalsBySlug. Kept for backward compat during transition. */ introsBySlug: Record; }; @@ -27,15 +30,22 @@ export default function cookbooksPlugin(context: LoadContext): Plugin { const contentSlugs = getCookbookSlugs(context.siteDir); assertCookbookSlugParity(contentSlugs); + const goalsBySlug: Record = {}; const introsBySlug: Record = {}; for (const slug of contentSlugs) { + const goal = readCookbookGoal(context.siteDir, slug); const intro = readCookbookIntro(context.siteDir, slug); - if (intro) { - introsBySlug[slug] = intro; + const text = goal ?? intro; + if (text) { + goalsBySlug[slug] = text; + introsBySlug[slug] = text; } } - actions.setGlobalData({ introsBySlug } satisfies CookbooksGlobalData); + actions.setGlobalData({ + goalsBySlug, + introsBySlug, + } satisfies CookbooksGlobalData); }, }; } diff --git a/src/lib/content-sections.ts b/src/lib/content-sections.ts index 0de79d9..ce2f9b1 100644 --- a/src/lib/content-sections.ts +++ b/src/lib/content-sections.ts @@ -7,7 +7,10 @@ const CONTENT_SECTION_FILES = [ ] as const; export type ContentSectionFile = (typeof CONTENT_SECTION_FILES)[number]; -/** Required file in every content folder — without it the slug is not published. */ +/** + * Legacy constant for backward compat. Slug detection in content-markdown.ts + * now accepts either goal.md or content.md as the required file. + */ export const REQUIRED_CONTENT_SECTION_FILE: ContentSectionFile = "content"; export type ContentSections = { @@ -37,14 +40,3 @@ export function joinContentSections(sections: ContentSections): string { ].filter((part): part is string => Boolean(part && part.trim())); return parts.map((part) => part.trim()).join("\n\n"); } - -/** Joins goal + content for human pages. Falls back to joinContentSections(). */ -export function joinGoalAndContent(sections: ContentSections): string { - const parts = [ - sections.goal, - sections.prerequisites, - sections.content, - sections.deployment, - ].filter((part): part is string => Boolean(part && part.trim())); - return parts.map((part) => part.trim()).join("\n\n"); -} diff --git a/src/lib/use-cookbook-markdown.ts b/src/lib/use-cookbook-markdown.ts index b836541..d680ff8 100644 --- a/src/lib/use-cookbook-markdown.ts +++ b/src/lib/use-cookbook-markdown.ts @@ -1,7 +1,7 @@ import { cookbooks, recipes, type Cookbook } from "@/lib/recipes/recipes"; import { useAllRecipeSections, - useCookbookIntro, + useCookbookGoal, } from "@/lib/use-raw-content-markdown"; import { composeCookbookMarkdown } from "@/lib/cookbook-composition"; @@ -11,10 +11,9 @@ type UseCookbookMarkdownResult = { }; /** - * Resolves a cookbook by id and assembles its agent-ready markdown by joining - * each child recipe's sections via `composeCookbookMarkdown`. Throws on - * missing cookbook, recipe, or recipe sections so config typos surface at - * page render time rather than producing silently empty exports. + * Resolves a cookbook by id and assembles its agent-ready markdown using + * goal-only mode: the cookbook's goal.md + each recipe's goal.md as + * labeled components. Skills handle implementation. */ export function useCookbookMarkdown( cookbookId: string, @@ -23,7 +22,7 @@ export function useCookbookMarkdown( if (!cookbook) throw new Error(`Cookbook ${cookbookId} not found`); const sectionsBySlug = useAllRecipeSections(); - const intro = useCookbookIntro(cookbookId); + const goal = useCookbookGoal(cookbookId); const recipeInputs = cookbook.recipeIds.map((id) => { const recipe = recipes.find((r) => r.id === id); @@ -37,8 +36,9 @@ export function useCookbookMarkdown( const rawMarkdown = composeCookbookMarkdown({ cookbookName: cookbook.name, cookbookDescription: cookbook.description, - intro, + goal, recipes: recipeInputs, + mode: "agent", }); return { cookbook, rawMarkdown }; diff --git a/src/lib/use-raw-content-markdown.ts b/src/lib/use-raw-content-markdown.ts index cf72f7d..0b40748 100644 --- a/src/lib/use-raw-content-markdown.ts +++ b/src/lib/use-raw-content-markdown.ts @@ -34,9 +34,17 @@ export function useRawSolutionMarkdown(slug: string): string | undefined { } type CookbooksGlobalData = { + goalsBySlug: Record; introsBySlug: Record; }; +export function useCookbookGoal(slug: string): string | undefined { + const data = usePluginData( + "docusaurus-plugin-cookbooks", + ) as CookbooksGlobalData; + return data.goalsBySlug[slug]; +} + export function useCookbookIntro(slug: string): string | undefined { const data = usePluginData( "docusaurus-plugin-cookbooks", diff --git a/src/pages/templates/app-with-lakebase.tsx b/src/pages/templates/app-with-lakebase.tsx index f9675a1..7de8c7a 100644 --- a/src/pages/templates/app-with-lakebase.tsx +++ b/src/pages/templates/app-with-lakebase.tsx @@ -1,6 +1,7 @@ import type { ReactNode } from "react"; import { CookbookDetail } from "@/components/cookbooks/cookbook-detail"; import { useCookbookMarkdown } from "@/lib/use-cookbook-markdown"; +import Goal from "@site/content/cookbooks/app-with-lakebase/goal.md"; import LakebaseCreateInstancePrereqs from "@site/content/recipes/lakebase-create-instance/prerequisites.md"; import LakebaseCreateInstanceContent from "@site/content/recipes/lakebase-create-instance/content.md"; import LakebaseDataPersistencePrereqs from "@site/content/recipes/lakebase-data-persistence/prerequisites.md"; @@ -11,6 +12,7 @@ export default function AppWithLakebasePage(): ReactNode { return ( +

Prerequisites

diff --git a/src/pages/templates/genie-analytics-app.tsx b/src/pages/templates/genie-analytics-app.tsx index 8610a5b..ad4a3ee 100644 --- a/src/pages/templates/genie-analytics-app.tsx +++ b/src/pages/templates/genie-analytics-app.tsx @@ -1,6 +1,7 @@ import type { ReactNode } from "react"; import { CookbookDetail } from "@/components/cookbooks/cookbook-detail"; import { useCookbookMarkdown } from "@/lib/use-cookbook-markdown"; +import Goal from "@site/content/cookbooks/genie-analytics-app/goal.md"; import GenieConversationalAnalyticsPrereqs from "@site/content/recipes/genie-conversational-analytics/prerequisites.md"; import GenieConversationalAnalyticsContent from "@site/content/recipes/genie-conversational-analytics/content.md"; @@ -9,6 +10,7 @@ export default function GenieAnalyticsAppPage(): ReactNode { return ( +

Prerequisites


diff --git a/src/pages/templates/lakebase-off-platform.tsx b/src/pages/templates/lakebase-off-platform.tsx index f5a4271..e0c50a6 100644 --- a/src/pages/templates/lakebase-off-platform.tsx +++ b/src/pages/templates/lakebase-off-platform.tsx @@ -1,6 +1,7 @@ import type { ReactNode } from "react"; import { CookbookDetail } from "@/components/cookbooks/cookbook-detail"; import { useCookbookMarkdown } from "@/lib/use-cookbook-markdown"; +import Goal from "@site/content/cookbooks/lakebase-off-platform/goal.md"; import LakebaseCreateInstancePrereqs from "@site/content/recipes/lakebase-create-instance/prerequisites.md"; import LakebaseCreateInstanceContent from "@site/content/recipes/lakebase-create-instance/content.md"; import LakebaseOffPlatformEnvManagementPrereqs from "@site/content/recipes/lakebase-off-platform-env-management/prerequisites.md"; @@ -17,6 +18,7 @@ export default function LakebaseOffPlatformPage(): ReactNode { return ( +

Prerequisites

diff --git a/src/pages/templates/operational-data-analytics.tsx b/src/pages/templates/operational-data-analytics.tsx index 57639d1..d661169 100644 --- a/src/pages/templates/operational-data-analytics.tsx +++ b/src/pages/templates/operational-data-analytics.tsx @@ -1,6 +1,7 @@ import type { ReactNode } from "react"; import { CookbookDetail } from "@/components/cookbooks/cookbook-detail"; import { useCookbookMarkdown } from "@/lib/use-cookbook-markdown"; +import Goal from "@site/content/cookbooks/operational-data-analytics/goal.md"; import UnityCatalogSetupPrereqs from "@site/content/recipes/unity-catalog-setup/prerequisites.md"; import UnityCatalogSetupContent from "@site/content/recipes/unity-catalog-setup/content.md"; import LakebaseCreateInstancePrereqs from "@site/content/recipes/lakebase-create-instance/prerequisites.md"; @@ -19,6 +20,7 @@ export default function OperationalDataAnalyticsPage(): ReactNode { return ( +

Prerequisites

diff --git a/tests/markdown.test.ts b/tests/markdown.test.ts index 15f3e1d..a4adedd 100644 --- a/tests/markdown.test.ts +++ b/tests/markdown.test.ts @@ -92,7 +92,7 @@ describe("detail markdown resolver", () => { test("template markdown for cookbook uses agent mode (goals only, no prereqs or full content)", () => { const markdown = getDetailMarkdown("templates", "ai-chat-app"); // Agent mode: cookbook goal + recipe goals as components - expect(markdown).toContain("What you are building"); + expect(markdown).toContain("streaming AI chat app on Databricks"); expect(markdown).toContain("## Component:"); // Agent mode should NOT include prerequisites section expect(markdown).not.toContain("## Prerequisites"); @@ -100,7 +100,7 @@ describe("detail markdown resolver", () => { test("template markdown includes cookbook goal.md above recipe goals", () => { const markdown = getDetailMarkdown("templates", "ai-chat-app"); - const goalIdx = markdown.indexOf("## What you are building"); + const goalIdx = markdown.indexOf("streaming AI chat app on Databricks"); const componentIdx = markdown.indexOf("## Component:"); expect(goalIdx).toBeGreaterThanOrEqual(0); expect(goalIdx).toBeLessThan(componentIdx); From 351ea3d1638207aad956bb65e587e7feb452b68b Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Tue, 19 May 2026 11:13:47 +0200 Subject: [PATCH 5/6] fix: align recipe content.md with agent skills source of truth - sync-tables-autoscaling: fix CLI commands from `databricks database` (Provisioned) to `databricks postgres` (Autoscaling), update JSON payload fields, add UC catalog prerequisite and DABs warning - lakebase-data-persistence: simplify scaffold from 7 --set flags to 2 (branch + database), matching current CLI behavior - lakebase-drizzle-off-platform: replace manual pg.Pool with recommended @databricks/lakebase package (createLakebasePool), simplify migration script, keep manual approach as fallback section - spin-up-databricks-app: add smoke test selector update step before validation (default selectors fail on customized apps) - genie-conversational-analytics: align genie_space_name handling with skill (assign in targets instead of removing variable) - set-up-your-local-dev-environment: add OAuth-over-PAT warning per databricks-cli-auth skill guidance Co-authored-by: Isaac --- .../genie-conversational-analytics/content.md | 20 ++- .../lakebase-data-persistence/content.md | 23 ++- .../lakebase-drizzle-off-platform/content.md | 142 +++++++++--------- .../content.md | 2 + .../recipes/spin-up-databricks-app/content.md | 8 +- .../sync-tables-autoscaling/content.md | 39 +++-- 6 files changed, 126 insertions(+), 108 deletions(-) diff --git a/content/recipes/genie-conversational-analytics/content.md b/content/recipes/genie-conversational-analytics/content.md index 2ecc8d8..79ee0c9 100644 --- a/content/recipes/genie-conversational-analytics/content.md +++ b/content/recipes/genie-conversational-analytics/content.md @@ -53,27 +53,33 @@ databricks apps init \ **Warning: Fix generated `databricks.yml` before deploying** -The scaffold generates a `genie_space_name` variable and references it as `name: ${var.genie_space_name}`, but never assigns a value. `bundle deploy` will fail with _no value assigned to required variable genie_space_name_. +The scaffold generates a `genie_space_name` variable and references it as `name: ${var.genie_space_name}`, but never assigns a value in `targets:`. `bundle deploy` will fail with _no value assigned to required variable genie_space_name_. -Your `variables:` block should look like this after the fix — only `genie_space_id`, no `genie_space_name`: +Assign both variables in your `targets:` block: ```yaml variables: genie_space_id: - description: Default Genie Space ID + description: Genie Space ID + genie_space_name: + description: Genie Space name + +targets: + default: + variables: + genie_space_id: + genie_space_name: ``` -And the `genie_space` resource block should use a hardcoded label: +The `genie_space` resource block should reference both variables: ```yaml genie_space: - name: genie-space + name: ${var.genie_space_name} space_id: ${var.genie_space_id} permission: CAN_RUN ``` -The `name: genie-space` is an internal label used by `app.yaml` (`valueFrom: genie-space`), not the Genie space display title. - Skip to step 8 to deploy. --- diff --git a/content/recipes/lakebase-data-persistence/content.md b/content/recipes/lakebase-data-persistence/content.md index 9bb5f3f..4c06bdc 100644 --- a/content/recipes/lakebase-data-persistence/content.md +++ b/content/recipes/lakebase-data-persistence/content.md @@ -9,21 +9,18 @@ The code examples below use a generic `items` resource as a placeholder. Replace ### 1. New app: scaffold with the Lakebase feature ```bash -databricks apps init \ - --name \ - --version latest \ - --features=lakebase \ - --set 'lakebase.postgres.branch=projects//branches/production' \ - --set 'lakebase.postgres.database=projects//branches/production/databases/' \ - --set 'lakebase.postgres.databaseName=' \ - --set 'lakebase.postgres.endpointPath=projects//branches/production/endpoints/primary' \ - --set 'lakebase.postgres.host=' \ - --set 'lakebase.postgres.port=5432' \ - --set 'lakebase.postgres.sslmode=require' \ +databricks apps init --name --features lakebase \ + --set "lakebase.postgres.branch=" \ + --set "lakebase.postgres.database=" \ --run none --profile ``` -Use the values returned by `list-databases` and `list-endpoints`. The generated template currently requires all postgres fields together during non-interactive scaffolding. +Where `` is the full branch resource name (e.g. `projects//branches/`) and `` is the full database resource name (e.g. `projects//branches//databases/`). Get these from: + +```bash +databricks postgres list-branches projects/ --profile +databricks postgres list-databases projects//branches/ --profile +``` This scaffolds a complete app with Lakebase already wired up, including a sample CRUD app. Skip to step 3 to configure environment variables, then step 5 to deploy. @@ -37,7 +34,7 @@ The scaffolded Lakebase sample uses `lakebase` in route names and file paths to ### 2. Existing app: add Lakebase manually -The following changes match what `apps init --features=lakebase` generates. Apply them to an existing scaffolded AppKit app. +The following changes match what `apps init --features lakebase` generates. Apply them to an existing scaffolded AppKit app. > **Tip:** The code below may be outdated. To get the latest, clone `https://github.com/databricks/appkit` and look in the `template/` directory. Search for `{{if .plugins.lakebase}}` to find all lakebase-conditional files and blocks. Files entirely wrapped in that conditional are lakebase-only; shared files like `App.tsx` and `server.ts` contain conditional blocks you can extract. diff --git a/content/recipes/lakebase-drizzle-off-platform/content.md b/content/recipes/lakebase-drizzle-off-platform/content.md index ef15bb1..2729e25 100644 --- a/content/recipes/lakebase-drizzle-off-platform/content.md +++ b/content/recipes/lakebase-drizzle-off-platform/content.md @@ -1,51 +1,31 @@ ## Drizzle ORM with Lakebase in an Off-Platform App -Connect Drizzle ORM to Lakebase in any Node.js server outside Databricks App Platform. Uses a `pg` Pool with a password callback for automatic credential refresh. +Connect Drizzle ORM to Lakebase in any Node.js server outside Databricks App Platform. Uses the `@databricks/lakebase` package for automatic OAuth token refresh. -### 1. Install Drizzle and the node-postgres driver +### 1. Install Drizzle and the Lakebase package ```bash -npm install drizzle-orm pg -npm install -D drizzle-kit @types/pg tsx +npm install drizzle-orm @databricks/lakebase +npm install -D drizzle-kit tsx ``` `drizzle-orm` and `drizzle-kit` must be on the same major version. If `drizzle-kit` errors with "This version of drizzle-kit is outdated," check that both packages share the same major (e.g. both 0.x or both 1.x). -### 2. Create a Lakebase-backed `pg` pool +### 2. Create a Lakebase-backed pool and Drizzle client -Create `src/lib/db/pool.ts`: +Create `src/lib/db/client.ts`. `createLakebasePool()` reads env vars automatically (`PGHOST`, `PGDATABASE`, `LAKEBASE_ENDPOINT`, `PGUSER`, etc.) and handles OAuth token refresh with a 2-minute buffer: ```typescript -import { Pool, type PoolConfig } from "pg"; -import { env } from "@/lib/env"; -import { getLakebasePostgresToken } from "@/lib/lakebase/tokens"; - -function sslConfig(mode: "require" | "prefer" | "disable"): PoolConfig["ssl"] { - switch (mode) { - case "require": - return { rejectUnauthorized: true }; - case "prefer": - return { rejectUnauthorized: false }; - case "disable": - return false; - } -} +import { drizzle } from "drizzle-orm/node-postgres"; +import { createLakebasePool } from "@databricks/lakebase"; +import * as itemsSchema from "@/lib/items/schema"; -export function createLakebasePool(): Pool { - return new Pool({ - host: env.PGHOST, - port: env.PGPORT, - database: env.PGDATABASE, - user: env.PGUSER, - password: () => getLakebasePostgresToken(), - ssl: sslConfig(env.PGSSLMODE), - max: 10, - idleTimeoutMillis: 30_000, - connectionTimeoutMillis: 10_000, - }); -} +const pool = createLakebasePool(); +export const db = drizzle({ client: pool, schema: { ...itemsSchema } }); ``` +> `@databricks/lakebase` is for **Lakebase Autoscaling only** (not compatible with Provisioned). See the manual alternative at the end if you need Provisioned support. + ### 3. Define a Drizzle schema Create `src/lib/items/schema.ts` with a starter table. Adapt the table name, columns, and types to your domain (e.g. `products`, `orders`, `users`): @@ -64,53 +44,25 @@ export const items = pgTable("items", { Add more schema files under `src/lib//schema.ts` as your app grows. The `drizzle.config.ts` glob (`./src/lib/*/schema.ts`) picks them all up automatically. -### 4. Initialize Drizzle with the pool +### 4. Write the migration script -Create `src/lib/db/client.ts`. Import every domain schema and spread it into the `schema` option: +Create `scripts/db-migrate.ts`. This uses the same `createLakebasePool()` with automatic credential handling — no need to build a temporary `DATABASE_URL`: ```typescript import { drizzle } from "drizzle-orm/node-postgres"; -import { createLakebasePool } from "@/lib/db/pool"; -import * as itemsSchema from "@/lib/items/schema"; +import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { createLakebasePool } from "@databricks/lakebase"; const pool = createLakebasePool(); -export const db = drizzle({ client: pool, schema: { ...itemsSchema } }); +const db = drizzle({ client: pool }); +await migrate(db, { migrationsFolder: "./src/lib/db/migrations" }); +await pool.end(); +console.log("Migrations applied successfully"); ``` -### 5. Handle drizzle-kit migrations with a temporary `DATABASE_URL` +### 5. Keep `drizzle.config.ts` minimal -`drizzle-kit` needs a connection string and cannot use `pg` password callbacks. Build a one-time URL with a fresh Lakebase credential in `scripts/db-migrate.ts`: - -```typescript -import { execSync } from "node:child_process"; -import { env } from "@/lib/env"; -import { getLakebasePostgresToken } from "@/lib/lakebase/tokens"; - -async function runMigrations() { - const token = await getLakebasePostgresToken(); - const encodedUser = encodeURIComponent(env.PGUSER); - const encodedPassword = encodeURIComponent(token); - - const databaseUrl = - `postgresql://${encodedUser}:${encodedPassword}` + - `@${env.PGHOST}:${env.PGPORT}/${env.PGDATABASE}` + - `?sslmode=${env.PGSSLMODE}`; - - execSync("npx drizzle-kit migrate", { - stdio: "inherit", - env: { ...process.env, DATABASE_URL: databaseUrl }, - }); -} - -runMigrations().catch((error) => { - console.error(error); - process.exit(1); -}); -``` - -### 6. Keep `drizzle.config.ts` minimal - -Lakebase Postgres passwords are short-lived tokens, so there is no static `DATABASE_URL` to store in `.env`. The migration script from step 5 builds a temporary URL with a fresh credential and passes it as `DATABASE_URL` when it shells out to `drizzle-kit migrate`. Commands like `generate` only read schema files and never connect, so `dbCredentials` is optional: +Commands like `generate` only read schema files and never connect, so no `dbCredentials` are needed: ```typescript import { defineConfig } from "drizzle-kit"; @@ -119,13 +71,10 @@ export default defineConfig({ schema: "./src/lib/*/schema.ts", out: "./src/lib/db/migrations", dialect: "postgresql", - ...(process.env.DATABASE_URL && { - dbCredentials: { url: process.env.DATABASE_URL }, - }), }); ``` -### 7. Verify schema generation and migration +### 6. Verify schema generation and migration Generate reads schema files locally (no database connection): @@ -143,7 +92,50 @@ npx dotenv -e .env.local -- npx tsx scripts/db-migrate.ts If both commands succeed, your Drizzle schema and Lakebase connection are working. +### Manual alternative (Provisioned or full control) + +If you cannot use `@databricks/lakebase` (e.g. Lakebase Provisioned, or you need full control over SSL and token refresh), build a manual `pg.Pool` with a password callback: + +```bash +npm install drizzle-orm pg +npm install -D drizzle-kit @types/pg tsx +``` + +```typescript +import { Pool, type PoolConfig } from "pg"; +import { env } from "@/lib/env"; +import { getLakebasePostgresToken } from "@/lib/lakebase/tokens"; + +function sslConfig(mode: "require" | "prefer" | "disable"): PoolConfig["ssl"] { + switch (mode) { + case "require": + return { rejectUnauthorized: true }; + case "prefer": + return { rejectUnauthorized: false }; + case "disable": + return false; + } +} + +export function createLakebasePool(): Pool { + return new Pool({ + host: env.PGHOST, + port: env.PGPORT, + database: env.PGDATABASE, + user: env.PGUSER, + password: () => getLakebasePostgresToken(), + ssl: sslConfig(env.PGSSLMODE), + max: 10, + idleTimeoutMillis: 30_000, + connectionTimeoutMillis: 10_000, + }); +} +``` + +For the migration script with this approach, build a temporary `DATABASE_URL` with a fresh credential and pass it to `drizzle-kit migrate` via `execSync`. + #### References - [Drizzle ORM with PostgreSQL](https://orm.drizzle.team/docs/get-started-postgresql) +- [`@databricks/lakebase` README](https://github.com/databricks/appkit/tree/main/packages/lakebase) - [Lakebase credentials API](https://docs.databricks.com/api/workspace/postgres/credentials) diff --git a/content/recipes/set-up-your-local-dev-environment/content.md b/content/recipes/set-up-your-local-dev-environment/content.md index 692c110..ec297d4 100644 --- a/content/recipes/set-up-your-local-dev-environment/content.md +++ b/content/recipes/set-up-your-local-dev-environment/content.md @@ -53,6 +53,8 @@ databricks -v ### 3. Authenticate a profile +Always use OAuth for local development. Personal Access Tokens (PATs) are for CI/CD or non-interactive environments only. + Browser-based OAuth is the default for local use: ```bash diff --git a/content/recipes/spin-up-databricks-app/content.md b/content/recipes/spin-up-databricks-app/content.md index 51fc1ac..a13c7e3 100644 --- a/content/recipes/spin-up-databricks-app/content.md +++ b/content/recipes/spin-up-databricks-app/content.md @@ -73,7 +73,11 @@ agent-browser open http://localhost:3000 Otherwise share the localhost URL with the user and ask them to click through the key flows. Do not deploy until the local app behaves as intended — Databricks Apps deploys are not free and a broken local build will not magically fix itself in production. -### 6. Validate and deploy +### 6. Update smoke tests + +Before validating, update `tests/smoke.spec.ts` to match your app's actual content. The default selectors look for "Minimal Databricks App" and "hello world" text, which will fail on any customized app. Use `getByRole`, `getByText`, or `getByPlaceholder` — never `getByLabelText` (that is a React Testing Library method, not Playwright). Keep result sets under the 1 MB analytics-event payload cap — queries returning thousands of rows cause `INVALID_REQUEST: Event exceeds max size`. Use `LIMIT` or aggregated queries. + +### 7. Validate and deploy Run the project validator first (build + typecheck + lint) so the deploy does not fail on something that would have been caught locally: @@ -89,7 +93,7 @@ databricks apps deploy --profile The CLI uploads the project, builds it on Databricks, and starts the app. On success it prints the workspace URL. -### 7. Verify the deployed app +### 8. Verify the deployed app ```bash databricks apps get --profile -o json diff --git a/content/recipes/sync-tables-autoscaling/content.md b/content/recipes/sync-tables-autoscaling/content.md index 971e37d..4136757 100644 --- a/content/recipes/sync-tables-autoscaling/content.md +++ b/content/recipes/sync-tables-autoscaling/content.md @@ -37,31 +37,48 @@ Autoscaling CUs are physically 8x smaller than Provisioned CUs, so per-CU throug ### 1. Create a synced table +> Your Lakebase database must be registered as a UC catalog first (one-time setup per project). If not already done: +> +> ```bash +> databricks postgres create-catalog \ +> --json '{ +> "spec": { +> "postgres_database": "", +> "branch": "projects//branches/" +> } +> }' --profile +> ``` + ```bash -databricks database create-synced-database-table \ +databricks postgres create-synced-table .. \ --json '{ - "name": "..", - "database_instance_name": "", - "logical_database_name": "", "spec": { "source_table_full_name": "..", "primary_key_columns": [""], "scheduling_policy": "", - "create_database_objects_if_missing": true + "branch": "projects//branches/", + "postgres_database": "databricks_postgres", + "create_database_objects_if_missing": true, + "new_pipeline_spec": { + "storage_catalog": "", + "storage_schema": "default" + } } }' --profile ``` -> If your Lakebase database is **registered as a Unity Catalog catalog**, you can omit `database_instance_name` and `logical_database_name`. +`new_pipeline_spec.storage_catalog` must be a **regular** UC catalog for DLT pipeline metadata, not the Lakebase catalog. Long-running operation; the CLI waits by default. Use `--no-wait` to return immediately. + +> **DABs:** Do not use `synced_database_tables` in DABs with Autoscaling projects — it maps to the Provisioned Terraform resource and may create unintended Provisioned instances. DAB support for Autoscaling synced tables is not yet available. Use the CLI commands above. Verify: ```bash -databricks database get-synced-database-table .. --profile +databricks postgres get-synced-table \ + "synced_tables/.." \ + --profile ``` -> **Important:** If your Autoscaling project was created via the `/postgres/` API (not `/database/`), programmatic synced table creation is not yet available via CLI. Use the Databricks UI as a fallback. In **Catalog**, select the source table → **Create synced table**, then choose your Lakebase project, branch, sync mode, and pipeline. This gap is expected to close soon. - ### 2. Configure pipeline reuse How you set up pipelines depends on your sync mode: @@ -80,8 +97,8 @@ The initial snapshot runs automatically on creation. For **Snapshot** and **Trig Trigger a sync update programmatically via the Databricks CLI. Look up the pipeline ID for the synced table, then start an update: ```bash -PIPELINE_ID=$(databricks database get-synced-database-table \ - .. \ +PIPELINE_ID=$(databricks postgres get-synced-table \ + "synced_tables/.." \ --output json --profile \ | jq -r '.data_synchronization_status.pipeline_id') From eeb9eab34059c41e7f385a068dafc629bc1b39d7 Mon Sep 17 00:00:00 2001 From: Pawel Kosiec Date: Tue, 19 May 2026 14:21:19 +0200 Subject: [PATCH 6/6] fix: cleanup stale comments, dead code, duplicated scaffold, and e2e tests - Remove stale JSDoc comment in content-markdown.ts - Remove dead useCookbookIntro (replaced by useCookbookGoal) - Deduplicate scaffold block in build-example-markdown.ts - Update cookbook validator hint to mention goal.md - Update e2e tests for goal-only agent prompt output Co-authored-by: Isaac --- scripts/validate-content.mjs | 2 +- src/lib/content-markdown.ts | 1 - src/lib/examples/build-example-markdown.ts | 48 +++++++--------------- src/lib/use-raw-content-markdown.ts | 7 ---- tests/e2e/copy-markdown.spec.ts | 37 +++++++---------- tests/e2e/navigation.spec.ts | 2 +- 6 files changed, 31 insertions(+), 66 deletions(-) diff --git a/scripts/validate-content.mjs b/scripts/validate-content.mjs index c285d00..2f612ae 100644 --- a/scripts/validate-content.mjs +++ b/scripts/validate-content.mjs @@ -142,7 +142,7 @@ if (existsSync(cookbooksDir)) { sectionPath: "content/cookbooks", sectionDir: cookbooksDir, allowedFiles: COOKBOOK_ALLOWED_FILES, - emptyHint: "Add at least intro.md or remove the folder.", + emptyHint: "Add goal.md or intro.md, or remove the folder.", flatHint: "Cookbook content lives under content/cookbooks//.", }); } diff --git a/src/lib/content-markdown.ts b/src/lib/content-markdown.ts index 4e0c612..6d3976b 100644 --- a/src/lib/content-markdown.ts +++ b/src/lib/content-markdown.ts @@ -29,7 +29,6 @@ export function hasSolutionSlug(rootDir: string, slug: string): boolean { return getSolutionSlugs(rootDir).includes(slug); } -/** Recipes and examples live in `content/
//` folders with a required content.md. */ /** * Recipes and examples live in `content/
//` folders. * A folder is published if it has either goal.md or content.md (or both). diff --git a/src/lib/examples/build-example-markdown.ts b/src/lib/examples/build-example-markdown.ts index 370831d..832d422 100644 --- a/src/lib/examples/build-example-markdown.ts +++ b/src/lib/examples/build-example-markdown.ts @@ -93,43 +93,25 @@ export function buildFullPrompt( lines.push("## Get started", ""); if (isInitCommand(example.initCommand)) { - if (!hasGoal) { - const hasPrereqs = Boolean(sections.prerequisites); - const hasDeployBlock = Boolean(sections.deployment); - - if (hasPrereqs) { - lines.push(sections.prerequisites!, ""); - } + if (!hasGoal && sections.prerequisites) { + lines.push(sections.prerequisites, ""); + } - lines.push( - "### Scaffold the project", - "", - "Run the command below to scaffold this example into a new directory using the [AppKit template system](/docs/appkit/v0/development/templates). It creates the app in your workspace, binds required resources, and writes a local `.env` with connection details resolved by the AppKit plugins.", - "", - "```bash", - example.initCommand, - "```", - "", - ); + lines.push( + "### Scaffold the project", + "", + "Run the command below to scaffold this example into a new directory using the [AppKit template system](/docs/appkit/v0/development/templates). It creates the app in your workspace, binds required resources, and writes a local `.env` with connection details resolved by the AppKit plugins.", + "", + "```bash", + example.initCommand, + "```", + "", + ); - if (hasDeployBlock) { - lines.push(sections.deployment!, ""); - } else { - lines.push( - "A **`README.md`** ships inside the scaffolded project. Follow it end to end to configure, run, and deploy the app.", - "", - ); - } + if (!hasGoal && sections.deployment) { + lines.push(sections.deployment, ""); } else { lines.push( - "### Scaffold the project", - "", - "Run the command below to scaffold this example into a new directory using the [AppKit template system](/docs/appkit/v0/development/templates). It creates the app in your workspace, binds required resources, and writes a local `.env` with connection details resolved by the AppKit plugins.", - "", - "```bash", - example.initCommand, - "```", - "", "A **`README.md`** ships inside the scaffolded project. Follow it end to end to configure, run, and deploy the app.", "", ); diff --git a/src/lib/use-raw-content-markdown.ts b/src/lib/use-raw-content-markdown.ts index 0b40748..14405b1 100644 --- a/src/lib/use-raw-content-markdown.ts +++ b/src/lib/use-raw-content-markdown.ts @@ -45,13 +45,6 @@ export function useCookbookGoal(slug: string): string | undefined { return data.goalsBySlug[slug]; } -export function useCookbookIntro(slug: string): string | undefined { - const data = usePluginData( - "docusaurus-plugin-cookbooks", - ) as CookbooksGlobalData; - return data.introsBySlug[slug]; -} - export function useExampleSections(slug: string): ContentSections | undefined { const data = usePluginData( "docusaurus-plugin-content-entries", diff --git a/tests/e2e/copy-markdown.spec.ts b/tests/e2e/copy-markdown.spec.ts index 8888418..1166fa1 100644 --- a/tests/e2e/copy-markdown.spec.ts +++ b/tests/e2e/copy-markdown.spec.ts @@ -43,7 +43,7 @@ async function clickCopyPromptAndWaitForToast( } test.describe("copy markdown exports raw markdown on recipe pages", () => { - test("recipe detail page copies actual markdown with code fences", async ({ + test("recipe detail page copies goal content with agent prompt wrapper", async ({ page, }) => { await setupClipboardMock(page); @@ -53,9 +53,8 @@ test.describe("copy markdown exports raw markdown on recipe pages", () => { const copied = await getCopiedText(page); expect(copied).toContain("# About DevHub"); - expect(copied).toContain("## Set Up Your Local Dev Environment"); - expect(copied).toContain("```bash"); - expect(copied).toContain("databricks -v"); + expect(copied).toContain("Install the Databricks CLI"); + expect(copied).toContain("authenticated CLI profile"); expect(copied).toContain("llms.txt"); }); }); @@ -75,14 +74,15 @@ test.describe("copy markdown exports raw markdown on template pages", () => { expect(copied).toContain("# Working with DevHub prompts"); expect(copied).toContain("# What the user just did"); expect(copied).toContain("# Verify your local Databricks dev environment"); - expect(copied).toContain("## Set Up Your Local Dev Environment"); + expect(copied).toContain("Install the Databricks CLI"); // Cookbook body comes after the meta-prompt, with its own frontmatter: expect(copied).toContain('title: "AI Chat App"'); expect(copied).toContain("# The cookbook the user copied"); - expect(copied).toContain("```bash"); + // Agent mode: recipe goals as components + expect(copied).toContain("## Component:"); }); - test("multi-recipe cookbook body no longer embeds the local-dev-environment recipe", async ({ + test("multi-recipe cookbook body uses agent mode with component headings", async ({ page, }) => { await setupClipboardMock(page); @@ -92,13 +92,8 @@ test.describe("copy markdown exports raw markdown on template pages", () => { const copied = await getCopiedText(page); expect(copied).toContain("# About DevHub"); - // The local-dev-environment recipe heading is present exactly once — - // injected by the meta-prompt, NOT duplicated inside the cookbook body. - const bootstrapHeadings = copied.match( - /^## Set Up Your Local Dev Environment$/gm, - ); - expect(bootstrapHeadings?.length).toBe(1); - expect(copied).toContain("## Lakebase Data Persistence"); + // Agent mode: recipe goals appear as labeled components, not full content + expect(copied).toContain("## Component: Lakebase Data Persistence"); expect(copied).toContain("---"); }); }); @@ -112,7 +107,7 @@ test.describe("copy markdown exports raw markdown on example pages", () => { const copied = await getCopiedText(page); expect(copied).toContain("# About DevHub"); - expect(copied).toContain("## Agentic Support Console"); + expect(copied).toContain("# Agentic Support Console"); expect(copied).toContain("Data Flow"); expect(copied).toContain("Lakehouse Sync"); }); @@ -125,7 +120,7 @@ test.describe("copy markdown exports raw markdown on example pages", () => { const copied = await getCopiedText(page); expect(copied).toContain("# About DevHub"); - expect(copied).toContain("## SaaS Subscription Tracker"); + expect(copied).toContain("# SaaS Subscription Tracker"); expect(copied).toContain("Data Flow"); }); @@ -152,12 +147,10 @@ test.describe("copy markdown exports raw markdown on example pages", () => { expect(copied).toContain( "These **templates** informed how this example was built", ); - expect(copied).toContain( - "### 1. Clone locally and follow `template/README.md`", - ); + expect(copied).toContain("### Clone and follow `template/README.md`"); }); - test("Banner Copy prompt copies full prompt with bash and ### substeps", async ({ + test("Banner Copy prompt copies full prompt with bash and clone substeps", async ({ page, }) => { await setupClipboardMock(page); @@ -168,9 +161,7 @@ test.describe("copy markdown exports raw markdown on example pages", () => { const copied = await getCopiedText(page); expect(copied).toContain("# About DevHub"); expect(copied).toContain("\n---\n\n# "); - expect(copied).toContain( - "### 1. Clone locally and follow `template/README.md`", - ); + expect(copied).toContain("### Clone and follow `template/README.md`"); expect(copied).toContain("```bash"); expect(copied).toContain( "git clone --depth 1 https://github.com/databricks/devhub.git", diff --git a/tests/e2e/navigation.spec.ts b/tests/e2e/navigation.spec.ts index 9f539ee..0aaf7ea 100644 --- a/tests/e2e/navigation.spec.ts +++ b/tests/e2e/navigation.spec.ts @@ -175,7 +175,7 @@ test.describe("home page link navigation", () => { expect(finalCopiedText).toContain( "# Verify your local Databricks dev environment", ); - expect(finalCopiedText).toContain("## Set Up Your Local Dev Environment"); + expect(finalCopiedText).toContain("Install the Databricks CLI"); expect(finalCopiedText).toContain("dev.databricks.com"); expect(finalCopiedText).toContain("llms.txt"); });