From cd3f12e60d96f83c699aa863f62cff08b024a68d Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 22:00:23 +0200 Subject: [PATCH 1/9] feat(appkit): metric-view route skeleton + measures-only SQL (PR2 phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xavier loop: iteration 1 — Phase 1 walking skeleton. POST /api/analytics/metric/:key over the standard SSE envelope, SP lane only. Synchronous config-parse registration from config/queries/metric-views.json against the landed metricSourceSchema (single metricViews map; lane derived from executor). Measures-only buildMetricSql (SELECT MEASURE(m) AS m FROM [LIMIT n]) gated by MEASURE_NAME_PATTERN + assertSafeFqn — grammar gate only, no name allowlist. 503 METRIC_REGISTRY_LOAD_FAILED on malformed config, 404 on unknown key (generic public bodies; detail to telemetry). Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 285 ++++++++++++ .../appkit/src/plugins/analytics/metric.ts | 236 ++++++++++ .../plugins/analytics/tests/analytics.test.ts | 11 +- .../plugins/analytics/tests/metric.test.ts | 421 ++++++++++++++++++ .../appkit/src/plugins/analytics/types.ts | 46 ++ 5 files changed, 996 insertions(+), 3 deletions(-) create mode 100644 packages/appkit/src/plugins/analytics/metric.ts create mode 100644 packages/appkit/src/plugins/analytics/tests/metric.test.ts diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 4ad05cc3..eceb4b9c 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -29,6 +29,11 @@ import { Plugin, toPlugin } from "../../plugin"; import type { PluginManifest } from "../../registry"; import { queryDefaults } from "./defaults"; import manifest from "./manifest.json"; +import { + buildMetricSql, + loadMetricRegistry, + validateMetricRequest, +} from "./metric"; import { QueryProcessor } from "./query"; import { type ArrowCapability, @@ -41,6 +46,7 @@ import { type AnalyticsStreamMessage, type IAnalyticsConfig, type IAnalyticsQueryRequest, + type MetricRegistration, normalizeAnalyticsFormat, type WarehouseStatus, } from "./types"; @@ -116,6 +122,25 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { */ private _arrowCapability = new Map(); + /** + * Metric-view registry parsed from `config/queries/metric-views.json`, keyed + * by metric key. Loaded lazily on the first `/metric/:key` request and + * memoized (see {@link _getMetricRegistry}). Empty when no config is present + * — the metric-view path stays dormant until an app opts in. + */ + private metricRegistry: Record | null = null; + + /** + * Latched error from the most recent {@link loadMetricRegistry} attempt. + * `null` means the registry loaded cleanly (or `metric-views.json` was absent + * — also fine; metric views are opt-in). When non-null, every `/metric/:key` + * request returns 503 `METRIC_REGISTRY_LOAD_FAILED` so a broken config + * (unreadable file, invalid JSON, schema violation) surfaces as a clear + * server status rather than masquerading as a 404 for every key. The full + * reason goes to telemetry only. + */ + private metricRegistryLoadError: string | null = null; + constructor(config: IAnalyticsConfig) { super(config); this.config = config; @@ -137,6 +162,19 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { }, }); + // Metric-view route. Registered parallel to `/query`; measures a + // registered UC Metric View over the same SSE envelope. Dormant until + // `config/queries/metric-views.json` exists (no config → empty registry → + // 404, nothing executes). + this.route(router, { + name: "metric", + method: "post", + path: "/metric/:key", + handler: async (req: express.Request, res: express.Response) => { + await this._handleMetricRoute(req, res); + }, + }); + // Column-names fallback for very wide Arrow schemas whose names don't fit // in the `X-Appkit-Arrow-Columns` response header (see // `_setArrowColumnsHeader`). The client hits this with the statement id @@ -426,6 +464,253 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ); } + /** + * Lazily load and memoize the metric registry from + * `config/queries/metric-views.json`. + * + * A malformed config latches `metricRegistryLoadError` and yields an empty + * registry so the route can return a 503 (distinguishing a broken deployment + * from an unknown key, which is a 404). Loading is deferred to the first + * `/metric/:key` request rather than the constructor so apps that never adopt + * metric views pay no parse cost and a config error can't break plugin + * construction. + */ + private _getMetricRegistry(): Record { + if (this.metricRegistry === null) { + try { + this.metricRegistry = loadMetricRegistry(); + this.metricRegistryLoadError = null; + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn("Failed to load metric registry: %s", reason); + this.metricRegistry = {}; + this.metricRegistryLoadError = reason; + } + } + return this.metricRegistry; + } + + /** + * Handle metric-view execution requests (`POST /api/analytics/metric/:key`). + * + * Mirrors {@link _handleQueryRoute}'s JSON SSE path: the outer + * `executeStream` disables cache/retry and streams warehouse-readiness + * (`warehouse_status`) events, then the inner `execute` builds the metric SQL + * and delivers rows through {@link deliverJsonResult} as a `result` message. + * The `originalError` re-throw discipline preserves each error's structured + * `errorCode`/`clientMessage` for the SSE error payload. + * + * SP lane only in this phase — every registered metric runs as the service + * principal; OBO dispatch is wired in a later phase. + */ + async _handleMetricRoute( + req: express.Request, + res: express.Response, + ): Promise { + const { key } = req.params; + + logger.debug(req, "Executing metric: %s", key); + + const event = logger.event(req); + event?.setComponent("analytics", "executeMetric").setContext("analytics", { + metric_key: key, + plugin: this.name, + }); + + if (!key) { + res.status(400).json({ error: "metric key is required" }); + return; + } + + const registry = this._getMetricRegistry(); + + // Surface a registry-load failure on the route. Without this, a malformed + // metric-views.json would yield 404 for every key — identical to "key + // never registered" — and hide the deployment error. Full reason → + // telemetry only. + if (this.metricRegistryLoadError !== null) { + event?.setContext("analytics", { + metric_registry_load_error: this.metricRegistryLoadError, + }); + res.status(503).json({ + error: "Metric registry not available", + code: "METRIC_REGISTRY_LOAD_FAILED", + }); + return; + } + + const registration = registry[key]; + if (!registration) { + // Don't echo the user-supplied `key` back in the public response — + // confirming "metric X is not registered" lets a probe enumerate keys by + // elimination. The 404 status stays (useful for tooling); the body is + // generic and detail goes to telemetry only. + event?.setContext("analytics", { unknown_metric_key: key }); + res.status(404).json({ error: "Metric not found" }); + return; + } + + // Validate the body on the canonical error path. `validateMetricRequest` + // throws a `ValidationError` (400) whose message names only field paths, + // never raw values. + let request: ReturnType; + try { + request = validateMetricRequest(req.body ?? {}); + } catch (err) { + if (err instanceof AppKitError) { + res.status(err.statusCode).json({ error: err.message, code: err.code }); + return; + } + event?.setContext("analytics", { + unexpected_error: err instanceof Error ? err.message : String(err), + metric_key: key, + }); + logger.warn( + req, + "Unexpected throw during metric request validation for %s: %s", + key, + err instanceof Error ? err.message : String(err), + ); + res.status(400).json({ error: "Invalid request body" }); + return; + } + + // SP lane only in this phase: the default executor + shared cache key. OBO + // dispatch (asUser(req) + per-user key) arrives in a later phase. + const executor = this; + const executorKey = "sp"; + + const cacheConfig = { + ...queryDefaults.cache, + cacheKey: ["analytics:metric", key, JSON.stringify(request), executorKey], + }; + + // Cache/retry/timeout scoped to the SQL execution itself (inner `execute`) + // so the warehouse-readiness phase isn't retried and the generator value + // never leaks into the cache. + const sqlConfig: PluginExecuteConfig = { + ...queryDefaults, + cache: cacheConfig, + }; + + // Outer stream: no cache/retry — `executeStream` would otherwise wrap the + // generator factory and cache the generator object itself. Telemetry + + // trace context still apply. + const streamExecutionSettings: StreamExecutionSettings = { + default: { + cache: { enabled: false }, + retry: { enabled: false }, + }, + }; + + const startupTimeoutMs = + this.config.warehouseStartupTimeoutMs ?? + DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS; + const autoStartWarehouse = this.config.autoStartWarehouse ?? true; + + const self = this; + + await executor.executeStream( + res, + async function* ( + signal, + ): AsyncGenerator { + const workspaceClient = getWorkspaceClient(); + const warehouseId = await getWarehouseId(); + + // Stream warehouse-readiness updates as SSE events, then run SQL. + const readinessUpdates = streamCallbacks( + (emit) => + self.SQLClient.ensureWarehouseRunning( + workspaceClient, + warehouseId, + { + signal, + timeoutMs: startupTimeoutMs, + autoStart: autoStartWarehouse, + onStatus: emit, + }, + ), + ); + for await (const update of readinessUpdates) { + yield { + type: "warehouse_status", + status: { + state: update.state as WarehouseStatus["state"], + elapsedMs: update.elapsedMs, + }, + }; + } + + // `execute()` reduces a thrown error to `{ status, message }`, + // dropping the rich fields (`errorCode`, `clientMessage`). Capture the + // original here so we can re-throw it intact — the SSE error path + // (`StreamManager`) reads `errorCode`/`clientMessage` off it. + let originalError: unknown; + const sqlResult = await executor.execute( + async (sig) => { + try { + const { statement, parameters } = buildMetricSql( + registration, + request, + ); + const processedParams = + await self.queryProcessor.processQueryParams( + statement, + Object.keys(parameters).length > 0 ? parameters : undefined, + ); + // Reuse the query route's JSON delivery: INLINE JSON_ARRAY with + // an ARROW_STREAM-inline fallback, returning plain rows in a + // `result` message — byte-identical envelope to `/query`. + return await self._executeJsonArrayPath( + executor, + statement, + processedParams, + sig, + ); + } catch (err) { + originalError = err; + throw err; + } + }, + { default: sqlConfig }, + executorKey, + ); + + if (!sqlResult.ok) { + const msg = sqlResult.message; + const lower = msg.toLowerCase(); + if ( + lower.includes("operation was aborted") || + lower.includes("the request was aborted") || + lower.includes("statement was canceled") + ) { + const err = new DOMException( + lower.includes("canceled") ? msg : "The operation was aborted.", + "AbortError", + ); + throw err; + } + // Re-throw the original error so its structured `errorCode` and + // sanitized `clientMessage` survive to the SSE error payload. Fall + // back to a generic statement failure only if it wasn't an + // AppKitError. + if (originalError instanceof AppKitError) { + throw originalError; + } + const inner = msg.startsWith("Statement failed: ") + ? msg.slice("Statement failed: ".length) + : msg; + throw ExecutionError.statementFailed(inner); + } + + yield sqlResult.data as AnalyticsStreamMessage; + }, + streamExecutionSettings, + executorKey, + ); + } + /** * JSON_ARRAY SSE path. Delegates the disposition/format fallback to * {@link deliverJsonResult} (INLINE JSON_ARRAY → on `needs-arrow-inline`, diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts new file mode 100644 index 00000000..70870384 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -0,0 +1,236 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { SQLTypeMarker } from "shared"; +import { z } from "zod"; +// Canonical metric-source schema — the single source of truth for +// `metric-views.json`. Imported from the shared source directly (matching the +// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the +// same tree) so the runtime and the generated JSON schema validate identically. +import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; +import { ValidationError } from "../../errors"; +import { createLogger } from "../../logging/logger"; +import type { + IAnalyticsMetricRequest, + MetricLane, + MetricRegistration, +} from "./types"; + +const logger = createLogger("analytics:metric"); + +/** + * Default queries directory. Mirrors `AppManager`'s + * `path.resolve(process.cwd(), "config/queries")` so dev mode and production + * share a single source of truth for where metric config lives. + */ +const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +const METRIC_CONFIG_FILE = "metric-views.json"; + +/** + * Three-part UC FQN matcher. The registry is parsed against the landed + * `metricSourceSchema`, which already validates `source` via the composed + * `UC_THREE_PART_FQN_PATTERN`; this is a belt-and-suspenders runtime fence for + * any code path that constructs SQL from a `source` outside that parse (the FQN + * cannot be parameterized — it is interpolated into the SQL string). + */ +const FQN_PATTERN = + /^[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; + +/** + * Validate measure names before they are interpolated into `MEASURE()`. + * + * Measure names cannot be parameterized — they are SQL identifiers, not + * literals. This conservative identifier shape is the security boundary for + * the interpolated tokens: there is deliberately NO name allowlist, so a + * well-formed-but-unknown measure falls through to the warehouse and surfaces + * as a sanitized canonical error (parity with the raw `.sql` flow). + */ +const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** + * Map an entry's declared `executor` to the internal execution lane: + * - `"user"` → `"obo"` (per-user cache, on-behalf-of) + * - `"app_service_principal"` (default) → `"sp"` (shared cache) + */ +function laneFromExecutor( + executor: "app_service_principal" | "user", +): MetricLane { + return executor === "user" ? "obo" : "sp"; +} + +/** + * Read and validate `config/queries/metric-views.json` into a metric registry. + * + * Synchronous by design — registration is a pure config parse with no + * warehouse round-trip, no `DESCRIBE`, and no build-time metadata bundle. The + * single `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. + * + * Returns an empty registry when the file is absent: the metric-view path is + * additive and dormant until an app opts in by adding the config. A malformed + * file (unreadable, invalid JSON, or schema violation) throws — the caller + * latches the failure so the route can surface a 503 rather than masking a + * broken deployment as a 404 for every key. + */ +export function loadMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Record { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let raw: string; + try { + raw = fs.readFileSync(metricPath, "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return {}; + } + throw err; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, + ); + } + + const result = metricSourceSchema.safeParse(parsed); + if (!result.success) { + const issues = result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; "); + throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); + } + + const registry: Record = {}; + for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { + registry[key] = { + key, + source: entry.source, + lane: laneFromExecutor(entry.executor), + }; + } + + logger.debug( + "Loaded metric registry: %d entry(ies)", + Object.keys(registry).length, + ); + return registry; +} + +/** + * SQL identifier safety guard — the FQN ships in the SQL string (it cannot be + * parameterized) so we re-check the regex at construction time. + * + * The registry loader already enforces the FQN grammar via `metricSourceSchema`; + * this is a runtime fence for any future code path that constructs SQL outside + * of a parsed registry. + */ +function assertSafeFqn(fqn: string): void { + if (!FQN_PATTERN.test(fqn)) { + throw new Error( + `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, + ); + } +} + +/** + * Structural validation schema for the metric request body. + * + * The schema is **static** — any grammar-valid measure identifier is accepted; + * it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but-well-formed + * names are not rejected here; they reach the warehouse and surface as a + * sanitized canonical error. This is the measures-only shape; + * dimensions/filter/timeGrain are accepted structurally (their SQL is built in + * a later phase) but left permissive here. + */ +const metricRequestSchema = z + .object({ + measures: z + .array(z.string().min(1, "measure name cannot be empty")) + .min(1, "at least one measure is required"), + dimensions: z.array(z.string().min(1)).optional(), + filter: z.unknown().optional(), + timeGrain: z.string().min(1).optional(), + limit: z.number().int().positive().optional(), + format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), + }) + .strict(); + +/** + * Validate a `POST /api/analytics/metric/:key` request body against the static + * measures-only shape. Throws {@link ValidationError} (a 400 on the canonical + * error path) with the offending field paths; the raw values stay in telemetry + * context, never the public body. + */ +export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + const result = metricRequestSchema.safeParse(body); + if (!result.success) { + const fieldPaths = result.error.issues + .map((i) => i.path.join(".") || "(root)") + .join(", "); + throw new ValidationError( + fieldPaths.length > 0 + ? `Invalid metric request body (fields: ${fieldPaths})` + : "Invalid metric request body", + { context: { issues: result.error.issues } }, + ); + } + return result.data; +} + +/** + * Construct the measures-only metric SQL. + * + * Shape: + * + * SELECT MEASURE(m) AS m[, …] FROM [LIMIT n] + * + * Every measure is gated by {@link MEASURE_NAME_PATTERN} before it is + * interpolated (measures cannot be parameterized — they are SQL identifiers), + * and the FQN is re-checked by {@link assertSafeFqn}. No user-supplied string + * reaches the SQL string without passing a grammar gate. `dimensions`, + * `filter`, and `timeGrain` are ignored here — their SQL is built in a later + * phase. Returns `{ statement, parameters }` where `parameters` is the named + * bind-var dictionary the plugin's `query()` consumes (empty in this phase). + */ +export function buildMetricSql( + registration: MetricRegistration, + request: IAnalyticsMetricRequest, +): { + statement: string; + parameters: Record; +} { + assertSafeFqn(registration.source); + + if (request.measures.length === 0) { + throw new Error("buildMetricSql requires at least one measure."); + } + + for (const m of request.measures) { + if (!MEASURE_NAME_PATTERN.test(m)) { + throw new Error( + `Refusing to build SQL: measure "${m}" is not a valid identifier.`, + ); + } + } + + // Deterministic order so cache keys collapse semantically equivalent calls. + // Alias each measure to its plain name so result rows have keys matching the + // registered measure (`{ arr: 1234 }`) rather than the SQL-function + // serialization Databricks returns by default (`{ "measure(arr)": 1234 }`). + const measureClauses = [...request.measures] + .sort() + .map((m) => `MEASURE(${m}) AS ${m}`); + + const selectList = measureClauses.join(", "); + + const limitClause = + typeof request.limit === "number" && request.limit > 0 + ? ` LIMIT ${Math.floor(request.limit)}` + : ""; + + const statement = `SELECT ${selectList} FROM ${registration.source}${limitClause}`; + return { statement, parameters: {} }; +} diff --git a/packages/appkit/src/plugins/analytics/tests/analytics.test.ts b/packages/appkit/src/plugins/analytics/tests/analytics.test.ts index 35677792..aff24614 100644 --- a/packages/appkit/src/plugins/analytics/tests/analytics.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/analytics.test.ts @@ -90,18 +90,23 @@ describe("Analytics Plugin", () => { }); describe("injectRoutes", () => { - test("should register single POST route for queries", () => { + test("should register the query and metric POST routes", () => { const plugin = new AnalyticsPlugin(config); const { router } = createMockRouter(); plugin.injectRoutes(router); - // Only 1 POST route - asUser is determined by .obo.sql file convention - expect(router.post).toHaveBeenCalledTimes(1); + // Two POST routes: the SQL query route (asUser determined by the + // .obo.sql file convention) and the metric-view route. + expect(router.post).toHaveBeenCalledTimes(2); expect(router.post).toHaveBeenCalledWith( "/query/:query_key", expect.any(Function), ); + expect(router.post).toHaveBeenCalledWith( + "/metric/:key", + expect.any(Function), + ); }); test("/query/:query_key should return 400 when query_key is missing", async () => { diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts new file mode 100644 index 00000000..b6b529a6 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -0,0 +1,421 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + createMockRequest, + createMockResponse, + createMockRouter, + mockServiceContext, + setupDatabricksEnv, +} from "@tools/test-helpers"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { ServiceContext } from "../../../context/service-context"; +import { AnalyticsPlugin } from "../analytics"; +import { buildMetricSql, loadMetricRegistry } from "../metric"; +import type { IAnalyticsConfig, MetricRegistration } from "../types"; + +// Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s +// cache interceptor is a no-op pass-through (each request re-executes). +const { mockCacheStore, mockCacheInstance } = vi.hoisted(() => { + const store = new Map(); + const generateKey = (parts: unknown[], userKey: string): string => { + const { createHash } = require("node:crypto"); + const serialized = JSON.stringify([userKey, ...parts]); + return createHash("sha256").update(serialized).digest("hex"); + }; + const instance = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + getOrExecute: vi.fn( + async (key: unknown[], fn: () => Promise, userKey: string) => { + const cacheKey = generateKey(key, userKey); + if (store.has(cacheKey)) return store.get(cacheKey); + const result = await fn(); + store.set(cacheKey, result); + return result; + }, + ), + generateKey: vi.fn((parts: unknown[], userKey: string) => + generateKey(parts, userKey), + ), + }; + return { mockCacheStore: store, mockCacheInstance: instance }; +}); + +vi.mock("../../../cache", () => ({ + CacheManager: { + getInstanceSync: vi.fn(() => mockCacheInstance), + }, +})); + +/** Feed the plugin a registry directly, bypassing the disk config parse. */ +function setRegistry( + plugin: AnalyticsPlugin, + registry: Record, +): void { + (plugin as any).metricRegistry = registry; + (plugin as any).metricRegistryLoadError = null; +} + +describe("analytics metric route (Phase 1)", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + describe("injectRoutes", () => { + test("registers POST /metric/:key alongside /query", () => { + const plugin = new AnalyticsPlugin(config); + const { router } = createMockRouter(); + + plugin.injectRoutes(router); + + expect(router.post).toHaveBeenCalledWith( + "/metric/:key", + expect.any(Function), + ); + }); + }); + + // ── Grammar gate — the primary security test (replaces #341's + // fail-closed-503 allowlist test). A measure that fails MEASURE_NAME_PATTERN + // throws inside buildMetricSql BEFORE any SQL string is constructed. + describe("buildMetricSql grammar gate", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("throws before building SQL for an injection-shaped measure", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr; DROP TABLE users"], + }), + ).toThrow(/not a valid identifier/); + }); + + test("throws for a measure with a backtick / quote", () => { + expect(() => + buildMetricSql(registration, { measures: ["arr`"] }), + ).toThrow(/not a valid identifier/); + }); + + test("throws when no measures are supplied", () => { + expect(() => buildMetricSql(registration, { measures: [] })).toThrow( + /at least one measure/, + ); + }); + + test("throws for a non-three-part source FQN", () => { + expect(() => + buildMetricSql( + { key: "x", source: "cat.sch", lane: "sp" }, + { measures: ["arr"] }, + ), + ).toThrow(/not a valid three-part UC FQN/); + }); + }); + + // ── Measures-only SQL shape. + describe("buildMetricSql SQL shape", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("single measure → SELECT MEASURE(m) AS m FROM ", () => { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + ); + expect(parameters).toEqual({}); + }); + + test("multiple measures are sorted for a deterministic SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["revenue", "arr"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM cat.sch.revenue_metrics", + ); + }); + + test("positive limit appends a floored LIMIT clause", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + limit: 10.9, + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics LIMIT 10", + ); + }); + }); + + // ── Envelope parity — streams warehouse_status* then a `result` message, + // byte-identical to the /query route's JSON SSE path. + describe("_handleMetricRoute SSE envelope", () => { + test("streams warehouse_status then a result message with aliased rows", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const executeMock = vi.fn().mockResolvedValue({ + result: { data: [{ arr: 1234 }] }, + }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The measures-only SQL reaches the warehouse. + expect(executeMock).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + statement: "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + warehouse_id: "test-warehouse-id", + }), + expect.any(AbortSignal), + ); + + // Same SSE envelope as /query: a `result` event carrying the rows. + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + expect(mockRes.write).toHaveBeenCalledWith( + expect.stringContaining('"data":[{"arr":1234}]'), + ); + expect(mockRes.end).toHaveBeenCalled(); + }); + + test("emits warehouse_status before result for a STARTING warehouse", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const executeMock = vi.fn().mockResolvedValue({ + result: { data: [{ arr: 1 }] }, + }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + const warehouseGet = vi + .fn() + .mockResolvedValueOnce({ state: "STARTING" }) + .mockResolvedValueOnce({ state: "RUNNING" }); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + mockReq.serviceWorkspaceClient.warehouses.get = warehouseGet; + mockReq.userWorkspaceClient.warehouses.get = warehouseGet; + const mockRes = createMockResponse(); + + vi.useFakeTimers(); + const handlerPromise = handler(mockReq, mockRes); + await vi.runAllTimersAsync(); + await handlerPromise; + vi.useRealTimers(); + + const eventLines = (mockRes.write as any).mock.calls + .map((call: any[]) => call[0] as string) + .filter((s: string) => s.startsWith("event: ")); + const warehouseIdx = eventLines.findIndex( + (s: string) => s === "event: warehouse_status\n", + ); + const resultIdx = eventLines.findIndex( + (s: string) => s === "event: result\n", + ); + expect(warehouseIdx).toBeGreaterThanOrEqual(0); + expect(resultIdx).toBeGreaterThanOrEqual(0); + expect(warehouseIdx).toBeLessThan(resultIdx); + }); + + test("returns 400 when the body fails structural validation", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: [] }, // empty → fails .min(1) + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(400); + }); + }); + + // ── 503-vs-404 latching + dormancy. + describe("registry latching and dormancy", () => { + test("unknown key against a valid registry → 404 (generic body)", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "nope" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(mockRes.json).toHaveBeenCalledWith({ error: "Metric not found" }); + }); + + test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + // Latch a load error directly (as a malformed config would). + (plugin as any).metricRegistry = {}; + (plugin as any).metricRegistryLoadError = "Invalid metric-views.json"; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(503); + expect(mockRes.json).toHaveBeenCalledWith({ + error: "Metric registry not available", + code: "METRIC_REGISTRY_LOAD_FAILED", + }); + }); + + test("no metric-views.json present → registry empty, unknown key 404, nothing executes", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + // Registry lazily loads from cwd; no config/queries/metric-views.json in + // the test cwd → empty registry (dormant). + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(executeMock).not.toHaveBeenCalled(); + }); + }); +}); + +// ── loadMetricRegistry: config parse against the landed metricSourceSchema. +describe("loadMetricRegistry", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), "mv-registry-")); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + test("absent metric-views.json → empty registry (dormancy)", () => { + expect(loadMetricRegistry(dir)).toEqual({}); + }); + + test("derives lane from executor (default sp, user → obo)", () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { + revenue: { source: "cat.sch.revenue_metrics" }, + customers: { source: "cat.sch.customer_metrics", executor: "user" }, + }, + }), + ); + + const registry = loadMetricRegistry(dir); + expect(registry.revenue).toEqual({ + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }); + expect(registry.customers).toEqual({ + key: "customers", + source: "cat.sch.customer_metrics", + lane: "obo", + }); + }); + + test("malformed JSON throws", () => { + writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); + expect(() => loadMetricRegistry(dir)).toThrow(/Failed to parse/); + }); + + test("schema-invalid config throws", () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "not-a-three-part-fqn" } }, + }), + ); + expect(() => loadMetricRegistry(dir)).toThrow(/Invalid metric-views.json/); + }); +}); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index 95c987d1..c032e6cc 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -112,3 +112,49 @@ export interface AnalyticsQueryResponse { row_count: number; data: any[]; } + +// ──────────────────────────────────────────────────────────────────────────── +// Metric views — POST /api/analytics/metric/:key +// ──────────────────────────────────────────────────────────────────────────── + +/** + * Execution lane for a registered metric view, derived from the entry's + * `executor` in `metric-views.json`: + * - `"sp"` ← `executor: "app_service_principal"` — queried as the app + * service principal (cache shared across all users). + * - `"obo"` ← `executor: "user"` — queried on-behalf-of the requesting + * user (per-user cache). OBO dispatch is wired in a later phase. + */ +export type MetricLane = "sp" | "obo"; + +/** + * A single registered metric view, loaded from `config/queries/metric-views.json`. + * + * The registration carries only what the runtime needs to build and dispatch + * SQL: the metric `key`, the three-part UC FQN `source`, and the `lane`. There + * is intentionally NO build-time measure/dimension metadata here — the security + * boundary is the grammar gate plus parameterized values, not a name allowlist, + * so the runtime never enumerates known measures/dimensions. + */ +export interface MetricRegistration { + key: string; + source: string; + lane: MetricLane; +} + +/** + * Validated request body for `POST /api/analytics/metric/:key`. + * + * `measures` is required; `dimensions`, `filter`, and `timeGrain` are part of + * the wire shape but their SQL is not built yet (a later phase adds grouping, + * time-grain truncation, and the structured filter translator). `filter` is + * typed `unknown` until that phase pins down the recursive predicate shape. + */ +export interface IAnalyticsMetricRequest { + measures: string[]; + dimensions?: string[]; + filter?: unknown; + timeGrain?: string; + limit?: number; + format?: AnalyticsFormat; +} From cb7467dfbe550f2ba04f762d9929140914b21442 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 22:41:48 +0200 Subject: [PATCH 2/9] feat(appkit): metric dimensions + structured filter engine (PR2 phase 2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xavier loop: iteration 2 — Phase 2 dimensions + filter engine. GROUP BY ALL for dimensions; recursive 12-operator filter engine (equals/notEquals/in/notIn/gt/gte/lt/lte/contains/notContains/set/notSet) with every value bound as a :f_ named parameter — never interpolated. member/dimension gated by DIMENSION_NAME_PATTERN; AND/OR composition; depth capped at 8, enforced twice (iterative pre-Zod preCheckFilterDepth + renderer re-check). Static (registration-free) request schema: operator enum, per-operator value cardinality, group/values/limit caps. No name allowlist. timeGrain application deliberately held (parsed + grammar-gated, not yet applied) pending the grain-target decision — lands in phase 2b. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/metric.ts | 759 ++++++++++++++++- .../plugins/analytics/tests/metric.test.ts | 762 +++++++++++++++++- .../appkit/src/plugins/analytics/types.ts | 54 +- 3 files changed, 1543 insertions(+), 32 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index 70870384..e2b03c97 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import path from "node:path"; -import type { SQLTypeMarker } from "shared"; +import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; import { z } from "zod"; // Canonical metric-source schema — the single source of truth for // `metric-views.json`. Imported from the shared source directly (matching the @@ -11,7 +11,10 @@ import { ValidationError } from "../../errors"; import { createLogger } from "../../logging/logger"; import type { IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, MetricLane, + MetricPredicate, MetricRegistration, } from "./types"; @@ -46,6 +49,100 @@ const FQN_PATTERN = */ const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; +/** + * Dimension name pattern. Matches the identifier shape we accept for measures + * — column references cannot be parameterized in SQL, so they must be + * conservatively safe identifiers (no spaces, no quotes, no SQL operators). + * This grammar gate is the security boundary for interpolated dimension + * tokens: there is deliberately NO name allowlist, so a well-formed-but-unknown + * dimension falls through to the warehouse and surfaces as a sanitized + * canonical error. + */ +const DIMENSION_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + +/** + * Time-grain token shape. Accepted structurally so a hostile token can never + * reach the SQL string, but the grain is NOT applied to any SQL this phase — + * see the seam in {@link renderDimensionClause}. + */ +const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; + +/** + * The exact twelve filter operators allowed at v1. The runtime tuple is the + * server-side source of truth; the client-side type union + * `MetricFilterOperatorName` mirrors these names statically. + */ +const METRIC_FILTER_OPERATORS = [ + "equals", + "notEquals", + "in", + "notIn", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", + "set", + "notSet", +] as const satisfies readonly MetricFilterOperatorName[]; + +/** + * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — + * enough for any real BI filter UI, low enough that a hostile or malformed + * payload cannot stack-overflow the recursive validator or translator. + * + * The depth count is the number of nested `{ and }` / `{ or }` wrappers + * encountered while descending — leaf predicates do not count toward depth. + */ +const METRIC_FILTER_MAX_DEPTH = 8; + +/** + * Cardinality caps on user-controlled arrays. Closes the recurring + * `unbounded-request-parameters` finding: a hostile caller could otherwise + * send `values: [...10M items...]` and exhaust the validator + the named + * bind-var binding step. The limits below are deliberately generous — higher + * than any real BI UI would emit — so legitimate traffic never trips them. + */ +const METRIC_MEASURES_MAX = 50; +const METRIC_DIMENSIONS_MAX = 20; +const METRIC_FILTER_VALUES_MAX = 1000; +const METRIC_LIMIT_MAX = 100_000; + +/** + * Maximum number of children per AND/OR group node. Without this cap a single + * flat group like `{ and: [...10M empty objects...] }` would push tens of + * millions of frames onto the iterative pre-check's stack — OOM before + * validation even gets to Zod. The Zod schema enforces the same cap so the + * rejection point is consistent regardless of which validator catches it + * first. + */ +const METRIC_FILTER_GROUP_MAX = 100; + +/** Operators that require exactly one value. */ +const SINGLE_VALUE_OPERATORS = new Set([ + "equals", + "notEquals", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", +]); + +/** Operators that require at least one value. */ +const LIST_VALUE_OPERATORS = new Set(["in", "notIn"]); + +/** Operators that reject `values` entirely. */ +const NULL_OPERATORS = new Set(["set", "notSet"]); + +/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ +const STRING_OPERATORS = new Set([ + "contains", + "notContains", +]); + /** * Map an entry's declared `executor` to the internal execution lane: * - `"user"` → `"obo"` (per-user cache, on-behalf-of) @@ -138,33 +235,311 @@ function assertSafeFqn(fqn: string): void { /** * Structural validation schema for the metric request body. * - * The schema is **static** — any grammar-valid measure identifier is accepted; - * it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but-well-formed - * names are not rejected here; they reach the warehouse and surface as a - * sanitized canonical error. This is the measures-only shape; - * dimensions/filter/timeGrain are accepted structurally (their SQL is built in - * a later phase) but left permissive here. + * The schema is **static** — any grammar-valid measure/dimension identifier is + * accepted; it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but- + * well-formed names are not rejected here; they reach the warehouse and surface + * as a sanitized canonical error. The identifier grammar gate is enforced in + * {@link buildMetricSql}/{@link renderFilter} at interpolation time; the body + * schema stays structural (item shape, cardinality caps, operator enum, + * per-operator value cardinality, and AND/OR depth). + * + * `filter` is recursive (`Predicate | { and: [...] } | { or: [...] }`), built + * with `z.lazy`. `timeGrain` is accepted as a grammar-shaped token but its SQL + * application is deferred (see {@link renderDimensionClause}). */ + +/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ +const filterPredicateSchema: z.ZodType = z + .object({ + member: z + .string() + .min(1, { message: "filter predicate 'member' cannot be empty" }), + operator: z.string().min(1, { + message: "filter predicate 'operator' cannot be empty", + }) as z.ZodType, + values: z + .array(z.union([z.string(), z.number()])) + .max(METRIC_FILTER_VALUES_MAX, { + message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, + }) + .optional(), + }) + .strict(); + +/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ +const filterSchema: z.ZodType = z.lazy(() => + z.union([ + filterPredicateSchema, + z + .object({ + and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + z + .object({ + or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + ]), +); + const metricRequestSchema = z .object({ measures: z .array(z.string().min(1, "measure name cannot be empty")) - .min(1, "at least one measure is required"), - dimensions: z.array(z.string().min(1)).optional(), - filter: z.unknown().optional(), - timeGrain: z.string().min(1).optional(), - limit: z.number().int().positive().optional(), + .min(1, "at least one measure is required") + .max(METRIC_MEASURES_MAX, { + message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, + }), + dimensions: z + .array(z.string().min(1, "dimension name cannot be empty")) + .max(METRIC_DIMENSIONS_MAX, { + message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, + }) + .optional(), + filter: filterSchema.optional(), + // Grammar-shaped only: the token is validated for safety but its SQL is + // not built this phase (grain target TBD). + timeGrain: z + .string() + .min(1, { message: "timeGrain cannot be empty" }) + .regex(TIME_GRAIN_PATTERN, { + message: "timeGrain must match /^[a-z][a-z_]*$/", + }) + .optional(), + limit: z + .number() + .int({ message: "limit must be an integer" }) + .positive({ message: "limit must be positive" }) + .max(METRIC_LIMIT_MAX, { + message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, + }) + .optional(), format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), }) - .strict(); + .strict() + .superRefine((value, ctx) => { + if (value.filter != null) { + validateFilterTree(value.filter, ctx, ["filter"], 0); + } + }) as z.ZodType; + +/** + * Recursive Zod-time validator for the filter tree. + * + * Pushes structured issues into the refinement context with stable paths + * (`filter.and.0.or.2.member`, etc.) so the canonical 400 error carries + * actionable diagnostics. Keeps the registry-free concerns in one descent: + * + * 1. Operator is one of the twelve. + * 2. Per-operator value cardinality (single / list / null operators). + * 3. `contains`/`notContains` carry a string value. + * 4. AND/OR nesting depth cap. + * + * There is NO member-allowlist and NO op⇄dimension-type check — the member is + * grammar-gated at SQL-build time, and dimension types are not tracked (no + * registry metadata). Returns void; issues accumulate on `ctx`. + */ +function validateFilterTree( + node: MetricFilter, + ctx: z.RefinementCtx, + path: Array, + depth: number, +): void { + if (node === null || typeof node !== "object") { + // The base schema rejects this via the union, but stay defensive. + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: "filter node must be a Predicate or { and } / { or } group", + }); + return; + } + + if ("and" in node || "or" in node) { + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }); + return; + } + + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, groupKey], + message: `filter ${groupKey} group must be an array of predicates or nested groups`, + }); + return; + } + + // Reject empty `or` groups: an empty disjunction is vacuously false, which + // silently drops the surrounding intent. Empty `and` is OK (vacuously true + // → no constraint). Forcing the caller to omit the predicate entirely is + // the only unambiguous choice. + if (groupKey === "or" && children.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "or"], + message: "filter 'or' group must contain at least one predicate", + }); + return; + } + + children.forEach((child, idx) => { + validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); + }); + return; + } + + // Leaf predicate. The base schema already enforced shape; here we layer in + // the operator vocabulary + per-operator value cardinality. + const predicate = node as MetricPredicate; + + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "operator"], + message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, + }); + // No further checks meaningful when the operator is unknown. + return; + } + + const op = predicate.operator; + const values = predicate.values; + const valuesLen = values?.length ?? 0; + + if (NULL_OPERATORS.has(op)) { + if (values != null && valuesLen > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" must not carry values`, + }); + } + } else if (SINGLE_VALUE_OPERATORS.has(op)) { + if (valuesLen !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, + }); + } + } else if (LIST_VALUE_OPERATORS.has(op)) { + if (valuesLen < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires at least one value`, + }); + } + } + + // Op⇄value-type: string operators require a string value. This is a + // registry-free structural check (not an op⇄dimension-type check) — it + // converts what would otherwise be a synchronous throw in `renderPredicate` + // (rendered as a 500) into a clean 400 at validation time. + if (STRING_OPERATORS.has(op) && valuesLen > 0) { + const v = predicate.values?.[0]; + if (typeof v !== "string") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires a string value (got ${typeof v})`, + }); + } + } +} + +/** + * Iterative pre-parse depth check. Zod's union/object parsers walk the input + * recursively before our `superRefine` depth cap fires, so a deeply-nested + * `{ and: [{ and: [...] }] }` payload could stack-overflow during parse, never + * reaching the validator's depth check. This walk is iterative (explicit + * stack) and aborts as soon as `METRIC_FILTER_MAX_DEPTH` is exceeded, so a + * hostile payload of any size cannot drive the call stack. + * + * Walks BOTH `and` and `or` branches when both are present on the same node — + * Zod's `.strict()` rejects the multi-key shape downstream, but the pre-check + * has to inspect every branch Zod might recurse into (an earlier version used + * `else if` and was bypassed by `{ and: [], or: }`). + * + * Group-children breadth is also capped: a flat `{ and: [...10M items...] }` + * cannot push 10M frames onto the explicit stack here. Predicate leaves do NOT + * count toward depth — only nested `and` / `or` wrappers — matching the rule + * {@link validateFilterTree} enforces. + */ +function preCheckFilterDepth(filter: unknown): void { + if (filter == null || typeof filter !== "object") return; + const stack: Array<[unknown, number]> = [[filter, 0]]; + while (stack.length > 0) { + const popped = stack.pop(); + if (popped === undefined) continue; + const [node, depth] = popped; + if (node == null || typeof node !== "object") continue; + const obj = node as Record; + for (const groupKey of ["and", "or"] as const) { + const children = obj[groupKey]; + if (!Array.isArray(children)) continue; + if (children.length > METRIC_FILTER_GROUP_MAX) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, + }, + }, + ); + } + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }, + }, + ); + } + for (const child of children) { + stack.push([child, depth + 1]); + } + } + } +} /** * Validate a `POST /api/analytics/metric/:key` request body against the static - * measures-only shape. Throws {@link ValidationError} (a 400 on the canonical + * structured shape. Throws {@link ValidationError} (a 400 on the canonical * error path) with the offending field paths; the raw values stay in telemetry * context, never the public body. + * + * The recursion depth is bounded BEFORE Zod sees the input — the schema's own + * depth check fires inside `superRefine`, which only runs after Zod's recursive + * parse has already walked the tree on the call stack. */ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + if (body != null && typeof body === "object") { + preCheckFilterDepth((body as { filter?: unknown }).filter); + } const result = metricRequestSchema.safeParse(body); if (!result.success) { const fieldPaths = result.error.issues @@ -181,19 +556,35 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { } /** - * Construct the measures-only metric SQL. + * Construct the metric SQL. * * Shape: * - * SELECT MEASURE(m) AS m[, …] FROM [LIMIT n] + * SELECT MEASURE(m) AS m[, …][, …] + * FROM + * [WHERE ] + * [GROUP BY ALL] + * [LIMIT n] + * + * Notes: + * - Every measure and dimension is gated by {@link MEASURE_NAME_PATTERN} / + * {@link DIMENSION_NAME_PATTERN} before it is interpolated (column + * references cannot be parameterized — they are SQL identifiers), and the + * FQN is re-checked by {@link assertSafeFqn}. There is deliberately NO name + * allowlist — the grammar gate is the security boundary. No user-supplied + * string reaches the SQL string without passing a grammar gate. + * - `GROUP BY ALL` is added when at least one dimension is requested. UC + * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; + * `GROUP BY ALL` is the documented form that works without re-listing each + * dimension. + * - The `WHERE` clause is rendered from the recursive filter tree. Every value + * flows through Statement Execution's named bind-var path (`:f_`); no + * value is ever interpolated as a literal. + * - `timeGrain` currently has NO effect on the emitted SQL — see the seam in + * {@link renderDimensionClause}. * - * Every measure is gated by {@link MEASURE_NAME_PATTERN} before it is - * interpolated (measures cannot be parameterized — they are SQL identifiers), - * and the FQN is re-checked by {@link assertSafeFqn}. No user-supplied string - * reaches the SQL string without passing a grammar gate. `dimensions`, - * `filter`, and `timeGrain` are ignored here — their SQL is built in a later - * phase. Returns `{ statement, parameters }` where `parameters` is the named - * bind-var dictionary the plugin's `query()` consumes (empty in this phase). + * Returns `{ statement, parameters }` where `parameters` is the named bind-var + * dictionary the plugin's `query()` consumes. */ export function buildMetricSql( registration: MetricRegistration, @@ -216,6 +607,15 @@ export function buildMetricSql( } } + const dimensions = request.dimensions ?? []; + for (const d of dimensions) { + if (!DIMENSION_NAME_PATTERN.test(d)) { + throw new Error( + `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, + ); + } + } + // Deterministic order so cache keys collapse semantically equivalent calls. // Alias each measure to its plain name so result rows have keys matching the // registered measure (`{ arr: 1234 }`) rather than the SQL-function @@ -224,13 +624,320 @@ export function buildMetricSql( .sort() .map((m) => `MEASURE(${m}) AS ${m}`); - const selectList = measureClauses.join(", "); + const dimensionClauses = [...dimensions] + .sort() + .map((d) => renderDimensionClause(d, request.timeGrain)); + + const selectList = [...measureClauses, ...dimensionClauses].join(", "); + const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; const limitClause = typeof request.limit === "number" && request.limit > 0 ? ` LIMIT ${Math.floor(request.limit)}` : ""; - const statement = `SELECT ${selectList} FROM ${registration.source}${limitClause}`; - return { statement, parameters: {} }; + // Filter translation. Every value is bound through `:f_` named params; + // every column identifier is grammar-gated in `renderFilter`. Empty filter or + // no filter → no WHERE. + const parameters: Record = {}; + let whereClause = ""; + if (request.filter !== undefined) { + const fragment = renderFilter(request.filter, parameters, { + counter: 0, + depth: 0, + }); + if (fragment !== null && fragment.length > 0) { + whereClause = ` WHERE ${fragment}`; + } + } + + const statement = `SELECT ${selectList} FROM ${registration.source}${whereClause}${groupByClause}${limitClause}`; + return { statement, parameters }; +} + +/** + * Mutable counter / depth threaded through {@link renderFilter}. Fresh per + * `buildMetricSql` call, so two requests never share bind-var indexes. + */ +interface FilterRenderState { + counter: number; + depth: number; +} + +/** + * Recursively render a filter tree into a SQL fragment, pushing bind values + * into `params` keyed by `:f_` names. + * + * Returns `null` for an empty group (no WHERE clause needed). The caller's + * `buildMetricSql` only emits `WHERE` when this returns a non-null, non-empty + * fragment. Empty `and: []` collapses to null (SQL's vacuous-truth semantics + * for AND); empty `or: []` renders `1 = 0` — the validator rejects `or: []` + * before the SQL builder, but if it slips through we render vacuous-false + * rather than dropping the predicate silently (defense in depth so a future + * validator bypass cannot turn `or: []` into "match everything"). + * + * Defense-in-depth: even though the request body's filter has already been + * validated, every member name is re-checked against {@link + * DIMENSION_NAME_PATTERN} here. If validation is ever bypassed, the SQL + * constructor still refuses to interpolate an unknown-shaped identifier. + */ +function renderFilter( + node: MetricFilter, + params: Record, + state: FilterRenderState, +): string | null { + if (node === null || typeof node !== "object") { + throw new Error( + "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", + ); + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new Error( + `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, + ); + } + + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + if (groupKey === "or") { + return "1 = 0"; + } + return null; + } + + // Sort-before-hash discipline: within a group, predicate leaves are + // stable-sorted by (member, operator) before contributing to the rendered + // fragment, so semantically equivalent calls produce the same SQL string + // and (downstream) the same cache key. + const sortedChildren = sortFilterChildren(children); + + const fragments: string[] = []; + const childState: FilterRenderState = { + counter: state.counter, + depth: state.depth + 1, + }; + for (const child of sortedChildren) { + const rendered = renderFilter(child, params, childState); + if (rendered != null && rendered.length > 0) { + fragments.push(rendered); + } + } + state.counter = childState.counter; + + if (fragments.length === 0) return null; + if (fragments.length === 1) return fragments[0]; + const joiner = groupKey === "and" ? " AND " : " OR "; + return `(${fragments.join(joiner)})`; + } + + // Leaf predicate — grammar-gate the member and operator, then render. + const predicate = node as MetricPredicate; + + if (!DIMENSION_NAME_PATTERN.test(predicate.member)) { + throw new Error( + `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, + ); + } + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + throw new Error( + `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, + ); + } + + return renderPredicate(predicate, params, state); +} + +/** + * Stable-sort filter children inside an AND/OR group by `(member, operator)`. + * + * Predicates carry both fields and sort by their pair; nested groups sort + * after predicates and stay in their original relative order (a nested group + * is opaque from the outside — we cannot collapse it to a single key). This is + * the sort-before-hash invariant applied at the SQL-fragment level so + * downstream cache keys collapse semantically equivalent calls. + */ +function sortFilterChildren( + children: ReadonlyArray, +): MetricFilter[] { + const indexed = children.map((child, idx) => { + let key: string; + let isPredicate: boolean; + if ( + child !== null && + typeof child === "object" && + !("and" in child) && + !("or" in child) + ) { + const p = child as MetricPredicate; + key = `${p.member}${p.operator}`; + isPredicate = true; + } else { + // Nested groups don't have a single (member, operator) — keep their + // original index so multiple nested groups within the same parent remain + // stable relative to each other. + key = ""; + isPredicate = false; + } + return { child, idx, key, isPredicate }; + }); + + indexed.sort((a, b) => { + if (a.isPredicate && !b.isPredicate) return -1; + if (!a.isPredicate && b.isPredicate) return 1; + if (a.isPredicate && b.isPredicate) { + if (a.key < b.key) return -1; + if (a.key > b.key) return 1; + } + return a.idx - b.idx; + }); + + return indexed.map((entry) => entry.child); +} + +/** + * Translate a single predicate into a SQL fragment. + * + * Every value flows through a freshly-allocated `:f_` named bind var. + * Nothing from the request body is ever interpolated as a literal — the + * fragment carries identifiers (grammar-gated) and operators (whitelisted), + * then references the bind name for each value. + * + * `set` and `notSet` emit `IS NULL` / `IS NOT NULL` with no bind value. `in` + * and `notIn` emit `IN (:f_0, :f_1, ...)`. `contains` and `notContains` emit + * `LIKE :f_0` / `NOT LIKE :f_0` and pre-bind the value with `%` wrapping. + */ +function renderPredicate( + predicate: MetricPredicate, + params: Record, + state: FilterRenderState, +): string { + const col = predicate.member; + const op = predicate.operator; + const values = predicate.values ?? []; + + switch (op) { + case "equals": + return `${col} = ${bindValue(values[0], params, state)}`; + case "notEquals": + return `${col} <> ${bindValue(values[0], params, state)}`; + case "gt": + return `${col} > ${bindValue(values[0], params, state)}`; + case "gte": + return `${col} >= ${bindValue(values[0], params, state)}`; + case "lt": + return `${col} < ${bindValue(values[0], params, state)}`; + case "lte": + return `${col} <= ${bindValue(values[0], params, state)}`; + case "in": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} IN (${placeholders.join(", ")})`; + } + case "notIn": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} NOT IN (${placeholders.join(", ")})`; + } + case "contains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} LIKE ${bindLikeValue(raw, params, state)}`; + } + case "notContains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; + } + case "set": + return `${col} IS NOT NULL`; + case "notSet": + return `${col} IS NULL`; + default: { + // Exhaustiveness — the operator union is closed; if this is reached the + // operator vocabulary widened without updating the switch. + const _exhaustive: never = op; + throw new Error( + `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, + ); + } + } +} + +/** + * Allocate a fresh `:f_` bind name for `value`, push the typed marker into + * `params`, and return the placeholder string. Bumps the counter. + */ +function bindValue( + value: string | number | undefined, + params: Record, + state: FilterRenderState, +): string { + if (value === undefined) { + throw new Error( + "Refusing to build SQL: filter predicate is missing a required value.", + ); + } + const name = `f_${state.counter}`; + state.counter += 1; + if (typeof value === "number") { + params[name] = sqlHelpers.number(value); + } else if (typeof value === "string") { + params[name] = sqlHelpers.string(value); + } else { + throw new Error( + `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, + ); + } + return `:${name}`; +} + +/** + * Like {@link bindValue}, but wraps the value in `%...%` for `LIKE` / + * `NOT LIKE`. SQL wildcards in the user-supplied string remain in the value + * (matching the documented "contains" semantics) — escape-on-receive could be + * added later as an opt-in if customers request strict-substring matching. + */ +function bindLikeValue( + value: string, + params: Record, + state: FilterRenderState, +): string { + const name = `f_${state.counter}`; + state.counter += 1; + params[name] = sqlHelpers.string(`%${value}%`); + return `:${name}`; +} + +/** + * Render a single SELECT-list clause for a dimension. + * + * timeGrain application deferred — see PR2 grain-target decision; dimension + * renders bare for now. When the grain target is settled, a time-typed + * dimension will render as `date_trunc('', ) AS ` here; the + * `timeGrain` token is already accepted (grammar-shaped) upstream so the wire + * contract is stable across that change. + */ +function renderDimensionClause( + dim: string, + _timeGrain: string | undefined, +): string { + return dim; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index b6b529a6..8c310ec4 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -11,7 +11,11 @@ import { import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ServiceContext } from "../../../context/service-context"; import { AnalyticsPlugin } from "../analytics"; -import { buildMetricSql, loadMetricRegistry } from "../metric"; +import { + buildMetricSql, + loadMetricRegistry, + validateMetricRequest, +} from "../metric"; import type { IAnalyticsConfig, MetricRegistration } from "../types"; // Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s @@ -166,6 +170,109 @@ describe("analytics metric route (Phase 1)", () => { }); }); + // ── Phase 2: dimensions + GROUP BY ALL. No date_trunc — timeGrain is held + // this phase (grain target TBD), so dimensions render bare. + describe("buildMetricSql dimensions + GROUP BY", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("measures + dimensions → SELECT MEASURE(...), FROM GROUP BY ALL", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + + test("dimensions are sorted for a deterministic SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["segment", "region"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, region, segment FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + + test("measures sort before dimensions in the SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["revenue", "arr"], + dimensions: ["region"], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + + test("empty dimensions array → no GROUP BY (ungrouped)", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: [], + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + ); + }); + + test("dimensions + limit compose GROUP BY ALL then LIMIT", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region"], + limit: 100, + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL LIMIT 100", + ); + }); + + test("timeGrain is parsed but not yet applied (grain target TBD) — dimension renders bare, no date_trunc", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + }); + // The held-timeGrain seam: the grain does not alter the SQL. The + // dimension is emitted bare — NOT wrapped in date_trunc(). + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + expect(statement).not.toContain("date_trunc"); + }); + }); + + // ── Phase 2: dimension grammar gate. A dimension failing + // DIMENSION_NAME_PATTERN throws inside buildMetricSql before SQL is built. + describe("buildMetricSql dimension grammar gate", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("throws before building SQL for an injection-shaped dimension", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region; DROP TABLE users"], + }), + ).toThrow(/not a valid identifier/); + }); + + test("throws for a dimension with a backtick / quote", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region`"], + }), + ).toThrow(/not a valid identifier/); + }); + }); + // ── Envelope parity — streams warehouse_status* then a `result` message, // byte-identical to the /query route's JSON SSE path. describe("_handleMetricRoute SSE envelope", () => { @@ -419,3 +526,656 @@ describe("loadMetricRegistry", () => { expect(() => loadMetricRegistry(dir)).toThrow(/Invalid metric-views.json/); }); }); + +// ── Phase 2: the structured filter engine (translator + validator). +// Registry-free: names are grammar-gated, values are parameterized. No +// allowlist, no op⇄dimension-type check. +describe("metric — filter translator", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + // Render a filter via buildMetricSql and return the WHERE fragment + params. + function render(filter: unknown) { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + filter: filter as never, + }); + const match = statement.match(/ WHERE (.+?)( GROUP BY| LIMIT|$)/); + const where = match ? match[1] : null; + return { statement, where, parameters }; + } + + describe("operators (12 unit tests)", () => { + test("equals → ` = :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "equals", + values: ["EMEA"], + }); + expect(where).toBe("region = :f_0"); + expect(parameters).toEqual({ + f_0: { __sql_type: "STRING", value: "EMEA" }, + }); + }); + + test("notEquals → ` <> :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notEquals", + values: ["EMEA"], + }); + expect(where).toBe("region <> :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); + }); + + test("in → ` IN (:f_0, :f_1, ...)`", () => { + const { where, parameters } = render({ + member: "region", + operator: "in", + values: ["EMEA", "APAC", "AMER"], + }); + expect(where).toBe("region IN (:f_0, :f_1, :f_2)"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); + expect(parameters.f_1).toEqual({ __sql_type: "STRING", value: "APAC" }); + expect(parameters.f_2).toEqual({ __sql_type: "STRING", value: "AMER" }); + }); + + test("notIn → ` NOT IN (:f_0, :f_1, ...)`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notIn", + values: ["EMEA", "APAC"], + }); + expect(where).toBe("region NOT IN (:f_0, :f_1)"); + expect(Object.keys(parameters)).toHaveLength(2); + }); + + test("gt → ` > :f_0` (numeric value bound as INT)", () => { + const { where, parameters } = render({ + member: "deal_size", + operator: "gt", + values: [10000], + }); + expect(where).toBe("deal_size > :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "INT", value: "10000" }); + }); + + test("gte → ` >= :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "gte", + values: [5000], + }); + expect(where).toBe("deal_size >= :f_0"); + }); + + test("lt → ` < :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "lt", + values: [100], + }); + expect(where).toBe("deal_size < :f_0"); + }); + + test("lte → ` <= :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "lte", + values: [50000], + }); + expect(where).toBe("deal_size <= :f_0"); + }); + + test("contains → ` LIKE :f_0` (value wrapped in %...%)", () => { + const { where, parameters } = render({ + member: "region", + operator: "contains", + values: ["MEA"], + }); + expect(where).toBe("region LIKE :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%MEA%" }); + }); + + test("notContains → ` NOT LIKE :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notContains", + values: ["test"], + }); + expect(where).toBe("region NOT LIKE :f_0"); + expect(parameters.f_0).toEqual({ + __sql_type: "STRING", + value: "%test%", + }); + }); + + test("set → ` IS NOT NULL` (no bind)", () => { + const { where, parameters } = render({ + member: "region", + operator: "set", + }); + expect(where).toBe("region IS NOT NULL"); + expect(parameters).toEqual({}); + }); + + test("notSet → ` IS NULL` (no bind)", () => { + const { where, parameters } = render({ + member: "region", + operator: "notSet", + }); + expect(where).toBe("region IS NULL"); + expect(parameters).toEqual({}); + }); + }); + + describe("AND/OR composition", () => { + test("flat AND group renders predicates joined by AND", () => { + const { where, parameters } = render({ + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Enterprise"] }, + ], + }); + expect(where).toBe("(region = :f_0 AND segment = :f_1)"); + expect(parameters.f_0.value).toBe("EMEA"); + expect(parameters.f_1.value).toBe("Enterprise"); + }); + + test("flat OR group renders predicates joined by OR", () => { + const { where } = render({ + or: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "region", operator: "equals", values: ["APAC"] }, + ], + }); + expect(where).toBe("(region = :f_0 OR region = :f_1)"); + }); + + test("AND-of-OR composes nested groups", () => { + const { where } = render({ + and: [ + { member: "region", operator: "in", values: ["EMEA", "APAC"] }, + { + or: [ + { member: "segment", operator: "equals", values: ["Enterprise"] }, + { member: "deal_size", operator: "gt", values: [50000] }, + ], + }, + ], + }); + expect(where).toContain("(region IN ("); + expect(where).toContain(" AND "); + expect(where).toContain("OR"); + }); + + test("OR-of-AND composes nested groups", () => { + const { where } = render({ + or: [ + { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Enterprise"] }, + ], + }, + { member: "region", operator: "equals", values: ["APAC"] }, + ], + }); + expect(where).toMatch(/^\(.+ OR .+\)$/); + expect(where).toContain(" AND "); + }); + + test("empty `and: []` group emits no WHERE clause", () => { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + filter: { and: [] }, + }); + expect(statement).not.toContain("WHERE"); + expect(parameters).toEqual({}); + }); + + test("empty `or: []` renders 1 = 0 (defense in depth past the validator)", () => { + // The validator rejects `or: []` (see cardinality tests), but if a bypass + // ever reaches the renderer, an empty disjunction must render vacuous- + // false — never drop the predicate (which would mean "match everything"). + const { where } = render({ or: [] }); + expect(where).toBe("1 = 0"); + }); + }); + + describe("depth cap", () => { + test("rejects 9 levels of AND nesting at preCheckFilterDepth (before Zod)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 9; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).toThrowError(/fields:.*filter/); + }); + + test("accepts exactly 8 levels of AND nesting (validator)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 8; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).not.toThrow(); + }); + + test("renderFilter independently rejects nesting past the depth cap", () => { + // Second, independent depth enforcement: even if a payload bypasses the + // pre-Zod pre-check and Zod's superRefine, the SQL renderer re-checks the + // depth as defense in depth. Build 9-deep and call buildMetricSql + // directly (skipping validateMetricRequest). + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 9; i += 1) { + node = { and: [node] }; + } + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + filter: node as never, + }), + ).toThrow(/nesting exceeds the maximum depth/); + }); + + test("rejects pathologically deep filter without stack-overflow (pre-parse cap)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 10_000; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).toThrowError(/fields:.*filter/); + }); + + test("rejects deep `or` even when paired with empty `and` (else-if bypass guard)", () => { + let deepOr: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 10_000; i += 1) { + deepOr = { or: [deepOr] }; + } + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { and: [], or: [deepOr] } as never, + }), + ).toThrowError(/fields:.*filter/); + }); + + test("rejects breadth-DoS: a single group with too many children", () => { + const wide = Array.from({ length: 1000 }, () => ({ + member: "region", + operator: "equals" as const, + values: ["EMEA"], + })); + expect(() => + validateMetricRequest({ measures: ["arr"], filter: { and: wide } }), + ).toThrowError(/fields:.*filter/); + }); + }); + + describe("parameterization safety (no values in rendered SQL)", () => { + test("string values do not appear verbatim in the SQL string", () => { + const sneaky = "EMEA' OR '1'='1"; + const { statement } = render({ + member: "region", + operator: "equals", + values: [sneaky], + }); + expect(statement).not.toContain(sneaky); + expect(statement).toContain(":f_0"); + }); + + test("numeric values do not appear verbatim in the SQL string", () => { + const { statement } = render({ + member: "deal_size", + operator: "gt", + values: [987654321], + }); + expect(statement).not.toContain("987654321"); + expect(statement).toContain(":f_0"); + }); + + test("LIKE wildcard is the only value transformation; the original string is not in SQL", () => { + const { statement } = render({ + member: "region", + operator: "contains", + values: ["dangerous%"], + }); + expect(statement).not.toContain("dangerous"); + expect(statement).toContain(":f_0"); + }); + + test("IN values are individually bound (not concatenated)", () => { + const { statement, parameters } = render({ + member: "region", + operator: "in", + values: ["A", "B", "C"], + }); + expect(statement).not.toMatch(/region IN \([^:]/); + expect(Object.keys(parameters)).toHaveLength(3); + }); + + test("a filter member failing DIMENSION_NAME_PATTERN throws before SQL is built", () => { + expect(() => + render({ + member: "region; DROP TABLE foo --", + operator: "equals", + values: ["x"], + }), + ).toThrowError(/not a valid identifier/); + }); + }); + + describe("validator — operator + cardinality (registry-free)", () => { + test("rejects an unknown operator", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + member: "region", + operator: "startsWith" as never, + values: ["E"], + }, + }), + ).toThrowError(/fields:.*filter\.operator/); + }); + + test("rejects equals with zero values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "equals", values: [] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects equals with multiple values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "equals", values: ["A", "B"] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects in with empty values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "in", values: [] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects set with values (cardinality — must be absent)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "set", values: ["EMEA"] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("accepts set with no values", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "set" }, + }), + ).not.toThrow(); + }); + + test("accepts notSet with an empty values array", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "notSet", values: [] }, + }), + ).not.toThrow(); + }); + + test("rejects `contains` with a non-string value", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + member: "region", + operator: "contains", + values: [42 as unknown as string], + }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects a filter predicate with too many values (DoS guard)", () => { + const big = Array.from({ length: 2000 }, (_, i) => `v${i}`); + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "in", values: big }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects a bare-array filter (not a Predicate or { and }/{ or } group)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: [ + { member: "region", operator: "in", values: ["EMEA"] }, + ] as never, + }), + ).toThrow(); + }); + + test("rejects empty `or` group (empty disjunction is vacuously false)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { or: [] }, + }), + ).toThrowError(/fields:.*filter\.or/); + }); + + test("accepts empty `and` group (no constraint contributed)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { and: [] }, + }), + ).not.toThrow(); + }); + + test("rejects an unknown operator at depth (nested filter)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { + member: "segment", + operator: "matches" as never, + values: ["X"], + }, + ], + }, + }), + ).toThrowError(/fields:.*filter\.and\.1\.operator/); + }); + + test("well-formed-but-unknown member is NOT rejected locally (no allowlist)", () => { + // The dropped-allowlist posture: a grammar-valid member that isn't a + // real dimension passes validation AND SQL construction — it reaches the + // warehouse, which is the authority on whether the column exists. + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "ghost_column", operator: "equals", values: ["x"] }, + }), + ).not.toThrow(); + const { where } = render({ + member: "ghost_column", + operator: "equals", + values: ["x"], + }); + expect(where).toBe("ghost_column = :f_0"); + }); + }); + + describe("timeGrain (grammar-shaped, not yet applied)", () => { + test("a grammar-valid timeGrain is accepted structurally", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + }), + ).not.toThrow(); + }); + + test("a timeGrain failing TIME_GRAIN_PATTERN is rejected", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "MONTH; DROP TABLE t", + }), + ).toThrowError(/fields:.*timeGrain/); + }); + + test("timeGrain with no dimensions is accepted (held: no cross-field rule this phase)", () => { + // The #341 '400 when timeGrain set with no time-typed dimension' rule is + // deliberately NOT implemented — the grain target is TBD. The grammar + // shape is still enforced; the token simply does not alter SQL. + expect(() => + validateMetricRequest({ measures: ["arr"], timeGrain: "day" }), + ).not.toThrow(); + }); + }); + + describe("sort-before-hash (predicate ordering inside groups)", () => { + test("predicate order does not affect the rendered SQL within an AND group", () => { + const a = render({ + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Ent"] }, + ], + }); + const b = render({ + and: [ + { member: "segment", operator: "equals", values: ["Ent"] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + }); + expect(a.where).toBe(b.where); + }); + }); + + // The warehouse-authoritative unknown-name parity test (sanitized + // clientMessage/errorCode envelope) lands here because the Phase 1 harness + // can drive the metric route end-to-end and assert on the SSE error bytes. + describe("warehouse-authoritative unknown-name parity", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + test("a well-formed-but-unknown measure reaches the warehouse and surfaces a sanitized error envelope", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + // The warehouse rejects the unknown column. The raw text carries the + // offending name; the connector wraps it in an ExecutionError whose + // `.message` is server-log-only and whose `clientMessage` is sanitized. + const { ExecutionError } = await import("../../../errors/execution"); + const rawWarehouseText = + "[UNRESOLVED_COLUMN] A column with name `ghost_measure` cannot be resolved"; + const executeMock = vi + .fn() + .mockRejectedValue( + ExecutionError.statementFailed(rawWarehouseText, "UNRESOLVED_COLUMN"), + ); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + // grammar-valid, but not a real measure — NOT rejected locally. + body: { measures: ["ghost_measure"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The well-formed-unknown measure was NOT rejected locally — it reached + // the warehouse (parity with the raw `.sql` flow). + expect(executeMock).toHaveBeenCalled(); + expect(executeMock.mock.calls[0][1]).toEqual( + expect.objectContaining({ + statement: + "SELECT MEASURE(ghost_measure) AS ghost_measure FROM cat.sch.revenue_metrics", + }), + ); + + // The SSE error envelope carries a sanitized clientMessage + structured + // errorCode — the raw warehouse text (with the offending column name) + // never reaches the client payload (#329 scrubbing, inherited). + const errorWrites = (mockRes.write as any).mock.calls + .map((call: any[]) => call[0] as string) + .filter((s: string) => s.startsWith("data: ")); + const errorPayload = errorWrites.find((s: string) => + s.includes('"errorCode"'), + ); + expect(errorPayload).toBeTruthy(); + expect(errorPayload).toContain('"errorCode":"UNRESOLVED_COLUMN"'); + expect(errorPayload).toContain('"error":"Query execution failed"'); + expect(errorPayload).not.toContain("ghost_measure"); + expect(errorPayload).not.toContain("UNRESOLVED_COLUMN]"); + }); + }); +}); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index c032e6cc..8812020e 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -142,18 +142,62 @@ export interface MetricRegistration { lane: MetricLane; } +/** + * v1 filter operator vocabulary — exactly twelve names. The runtime tuple + * `METRIC_FILTER_OPERATORS` (next to the validator in `metric.ts`) is the + * server-side source of truth; this union mirrors it statically. + */ +export type MetricFilterOperatorName = + | "equals" + | "notEquals" + | "in" + | "notIn" + | "gt" + | "gte" + | "lt" + | "lte" + | "contains" + | "notContains" + | "set" + | "notSet"; + +/** + * A single filter predicate — the leaf node of the recursive + * {@link MetricFilter} tree. `member` is a dimension name (grammar-gated, not + * allowlisted); `values` is bound through parameterized `:f_` bind vars + * and never interpolated into the SQL string. + */ +export interface MetricPredicate { + member: string; + operator: MetricFilterOperatorName; + values?: ReadonlyArray; +} + +/** + * Recursive filter expression for the metric-view request body: a leaf + * {@link MetricPredicate} or an `{ and: [...] }` / `{ or: [...] }` group. The + * shape is intentionally non-generic server-side — per-metric narrowing (if + * any) lives client-side. + */ +export type MetricFilter = + | MetricPredicate + | { and: ReadonlyArray } + | { or: ReadonlyArray }; + /** * Validated request body for `POST /api/analytics/metric/:key`. * - * `measures` is required; `dimensions`, `filter`, and `timeGrain` are part of - * the wire shape but their SQL is not built yet (a later phase adds grouping, - * time-grain truncation, and the structured filter translator). `filter` is - * typed `unknown` until that phase pins down the recursive predicate shape. + * `measures` is required. `dimensions` drive `GROUP BY ALL`; `filter` is the + * recursive structured predicate tree translated into a parameterized `WHERE` + * clause. `timeGrain` is accepted structurally (grammar-shaped) but its SQL + * application is deferred — the runtime target of the grain is an open design + * question being resolved separately, so it currently does not alter the + * emitted SQL. */ export interface IAnalyticsMetricRequest { measures: string[]; dimensions?: string[]; - filter?: unknown; + filter?: MetricFilter; timeGrain?: string; limit?: number; format?: AnalyticsFormat; From 3584cabc410b0f97a42efce44337dff44ac40ebf Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 22:55:51 +0200 Subject: [PATCH 3/9] feat(appkit): apply timeGrain via explicit timeDimension (PR2 phase 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xavier loop: iteration 2 — Phase 2 grain slice. Resolves the grain-target gap: PR2 drops the registry that #341 used to infer which dimension was temporal, so the grain's target is named explicitly. New optional `timeDimension` request field; when `timeGrain` is set, that column renders as date_trunc('', ) AS (grain single-quoted literal gated by TIME_GRAIN_PATTERN; column gated by DIMENSION_NAME_PATTERN — neither can be a bind param). Other dimensions render bare. Two 400 rules in the request schema superRefine: timeGrain without timeDimension; timeDimension not in dimensions. Warehouse remains the authority on column temporality. Deviates from the PRD's "date_trunc on time-typed dimensions" wording (which presupposed the deleted registry) — the explicit-field shape is the runtime query shape PR2 settles; PR5's hook contract inherits the timeDimension arg. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/metric.ts | 92 +++++++++++--- .../plugins/analytics/tests/metric.test.ts | 116 +++++++++++++++--- .../appkit/src/plugins/analytics/types.ts | 14 ++- 3 files changed, 186 insertions(+), 36 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index e2b03c97..bce144cd 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -61,9 +61,10 @@ const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; const DIMENSION_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; /** - * Time-grain token shape. Accepted structurally so a hostile token can never - * reach the SQL string, but the grain is NOT applied to any SQL this phase — - * see the seam in {@link renderDimensionClause}. + * Time-grain token shape. This grammar gate is the security boundary for the + * grain: it is interpolated as a single-quoted `date_trunc` unit literal (NOT + * a bind param) in {@link renderDimensionClause}, so the pattern is what keeps + * a hostile token out of that quoted position. */ const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; @@ -244,8 +245,10 @@ function assertSafeFqn(fqn: string): void { * per-operator value cardinality, and AND/OR depth). * * `filter` is recursive (`Predicate | { and: [...] } | { or: [...] }`), built - * with `z.lazy`. `timeGrain` is accepted as a grammar-shaped token but its SQL - * application is deferred (see {@link renderDimensionClause}). + * with `z.lazy`. `timeGrain` buckets `timeDimension` via `date_trunc` (see + * {@link renderDimensionClause}); the two cross-field rules (grain requires + * timeDimension; timeDimension must be one of dimensions) live in the + * `superRefine` below. */ /** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ @@ -302,8 +305,10 @@ const metricRequestSchema = z }) .optional(), filter: filterSchema.optional(), - // Grammar-shaped only: the token is validated for safety but its SQL is - // not built this phase (grain target TBD). + // Grammar-shaped bucketing grain, applied to `timeDimension` via + // `date_trunc`. The token is validated for safety here; the grain literal + // is interpolated (single-quoted, never a bind param) in + // `renderDimensionClause`, so this pattern gate is the security boundary. timeGrain: z .string() .min(1, { message: "timeGrain cannot be empty" }) @@ -311,6 +316,17 @@ const metricRequestSchema = z message: "timeGrain must match /^[a-z][a-z_]*$/", }) .optional(), + // The single dimension `timeGrain` applies to via `date_trunc`. Grammar- + // gated as a SQL identifier (column reference, cannot be parameterized). + // Cross-field rules in `superRefine`: required when `timeGrain` is set, and + // must be one of `dimensions`. + timeDimension: z + .string() + .min(1, { message: "timeDimension cannot be empty" }) + .regex(DIMENSION_NAME_PATTERN, { + message: "timeDimension must match /^[a-zA-Z_][a-zA-Z0-9_]*$/", + }) + .optional(), limit: z .number() .int({ message: "limit must be an integer" }) @@ -326,6 +342,28 @@ const metricRequestSchema = z if (value.filter != null) { validateFilterTree(value.filter, ctx, ["filter"], 0); } + + // Cross-field grain rules. `timeGrain` buckets exactly one selected + // dimension via `date_trunc`, so it requires an explicit `timeDimension`, + // and that dimension must be selected (so it is in the SELECT list and + // `GROUP BY ALL`). + if (value.timeGrain != null && value.timeDimension == null) { + ctx.addIssue({ + code: "custom", + message: "timeDimension is required when timeGrain is set", + path: ["timeDimension"], + }); + } + if ( + value.timeDimension != null && + !(value.dimensions ?? []).includes(value.timeDimension) + ) { + ctx.addIssue({ + code: "custom", + message: "timeDimension must be one of dimensions", + path: ["timeDimension"], + }); + } }) as z.ZodType; /** @@ -580,8 +618,10 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { * - The `WHERE` clause is rendered from the recursive filter tree. Every value * flows through Statement Execution's named bind-var path (`:f_`); no * value is ever interpolated as a literal. - * - `timeGrain` currently has NO effect on the emitted SQL — see the seam in - * {@link renderDimensionClause}. + * - When `timeGrain` is set, the `timeDimension` column renders as + * `date_trunc('', ) AS ` (the grain is a grammar-gated + * single-quoted literal, not a bind param); other dimensions render bare — + * see {@link renderDimensionClause}. * * Returns `{ statement, parameters }` where `parameters` is the named bind-var * dictionary the plugin's `query()` consumes. @@ -626,7 +666,9 @@ export function buildMetricSql( const dimensionClauses = [...dimensions] .sort() - .map((d) => renderDimensionClause(d, request.timeGrain)); + .map((d) => + renderDimensionClause(d, request.timeGrain, request.timeDimension), + ); const selectList = [...measureClauses, ...dimensionClauses].join(", "); const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; @@ -929,15 +971,33 @@ function bindLikeValue( /** * Render a single SELECT-list clause for a dimension. * - * timeGrain application deferred — see PR2 grain-target decision; dimension - * renders bare for now. When the grain target is settled, a time-typed - * dimension will render as `date_trunc('', ) AS ` here; the - * `timeGrain` token is already accepted (grammar-shaped) upstream so the wire - * contract is stable across that change. + * When `timeGrain` is set and `dim` is the request's `timeDimension`, the + * column is bucketed: `date_trunc('', ) AS `. Every other + * dimension renders bare. Both the grain literal and the column identifier are + * grammar-gated before they reach the SQL string: + * + * - The grain is single-quoted (it is a `date_trunc` unit literal, NOT a bind + * param) and `TIME_GRAIN_PATTERN` upstream is the control that keeps a + * hostile token out of that quoted position. + * - The column cannot be parameterized (it is an identifier), so it is + * re-checked against {@link DIMENSION_NAME_PATTERN} here as belt-and- + * suspenders even though the schema already gated `timeDimension`. + * + * The aliasing keeps result-row keys stable (`{ order_date: ... }`) regardless + * of whether the grain was applied. */ function renderDimensionClause( dim: string, - _timeGrain: string | undefined, + timeGrain: string | undefined, + timeDimension: string | undefined, ): string { + if (timeGrain != null && dim === timeDimension) { + if (!DIMENSION_NAME_PATTERN.test(dim)) { + throw new Error( + `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, + ); + } + return `date_trunc('${timeGrain}', ${dim}) AS ${dim}`; + } return dim; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 8c310ec4..50efd534 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -170,8 +170,8 @@ describe("analytics metric route (Phase 1)", () => { }); }); - // ── Phase 2: dimensions + GROUP BY ALL. No date_trunc — timeGrain is held - // this phase (grain target TBD), so dimensions render bare. + // ── Phase 2: dimensions + GROUP BY ALL. Bare dimensions here; date_trunc + // grain application (via timeDimension) is covered in its own block below. describe("buildMetricSql dimensions + GROUP BY", () => { const registration: MetricRegistration = { key: "revenue", @@ -230,21 +230,57 @@ describe("analytics metric route (Phase 1)", () => { ); }); - test("timeGrain is parsed but not yet applied (grain target TBD) — dimension renders bare, no date_trunc", () => { + test("no timeGrain → all dimensions render bare (no date_trunc)", () => { const { statement } = buildMetricSql(registration, { measures: ["arr"], - dimensions: ["order_date"], - timeGrain: "month", + dimensions: ["order_date", "region"], }); - // The held-timeGrain seam: the grain does not alter the SQL. The - // dimension is emitted bare — NOT wrapped in date_trunc(). expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", ); expect(statement).not.toContain("date_trunc"); }); }); + // ── Phase 2a: timeGrain + timeDimension → date_trunc on the named column. + // The grain is a grammar-gated single-quoted literal; the column keeps its + // plain alias; other dimensions render bare; GROUP BY ALL is present. + describe("buildMetricSql timeGrain + timeDimension (date_trunc)", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("buckets only the timeDimension via date_trunc; other dimensions bare", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "order_date", + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + expect(statement).toContain( + "date_trunc('month', order_date) AS order_date", + ); + expect(statement).toContain(" GROUP BY ALL"); + }); + + test("the grain literal is threaded through (day) for the timeDimension", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + ); + }); + }); + // ── Phase 2: dimension grammar gate. A dimension failing // DIMENSION_NAME_PATTERN throws inside buildMetricSql before SQL is built. describe("buildMetricSql dimension grammar gate", () => { @@ -1049,13 +1085,14 @@ describe("metric — filter translator", () => { }); }); - describe("timeGrain (grammar-shaped, not yet applied)", () => { - test("a grammar-valid timeGrain is accepted structurally", () => { + describe("timeGrain + timeDimension (validator)", () => { + test("a grammar-valid timeGrain + timeDimension (in dimensions) is accepted", () => { expect(() => validateMetricRequest({ measures: ["arr"], dimensions: ["order_date"], timeGrain: "month", + timeDimension: "order_date", }), ).not.toThrow(); }); @@ -1066,17 +1103,64 @@ describe("metric — filter translator", () => { measures: ["arr"], dimensions: ["order_date"], timeGrain: "MONTH; DROP TABLE t", + timeDimension: "order_date", }), ).toThrowError(/fields:.*timeGrain/); }); - test("timeGrain with no dimensions is accepted (held: no cross-field rule this phase)", () => { - // The #341 '400 when timeGrain set with no time-typed dimension' rule is - // deliberately NOT implemented — the grain target is TBD. The grammar - // shape is still enforced; the token simply does not alter SQL. + test("a capitalized grain (Month) is rejected by the grammar gate", () => { expect(() => - validateMetricRequest({ measures: ["arr"], timeGrain: "day" }), - ).not.toThrow(); + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "Month", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeGrain/); + }); + + test("a timeDimension failing DIMENSION_NAME_PATTERN is rejected", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date; DROP TABLE t", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeGrain without timeDimension → 400 (grain requires a target)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "day", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeDimension not in dimensions → 400 (must be selectable + in GROUP BY ALL)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region"], + timeGrain: "month", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeDimension not in dimensions → 400 even without timeGrain", () => { + // The dimensions-membership rule holds independent of timeGrain: a + // timeDimension that isn't selected can never be bucketed or grouped. + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region"], + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeDimension/); }); }); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index 8812020e..e91a361f 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -189,16 +189,22 @@ export type MetricFilter = * * `measures` is required. `dimensions` drive `GROUP BY ALL`; `filter` is the * recursive structured predicate tree translated into a parameterized `WHERE` - * clause. `timeGrain` is accepted structurally (grammar-shaped) but its SQL - * application is deferred — the runtime target of the grain is an open design - * question being resolved separately, so it currently does not alter the - * emitted SQL. + * clause. `timeGrain` buckets the single dimension named by `timeDimension` + * via `date_trunc`; it requires `timeDimension`, and `timeDimension` must be + * one of `dimensions` so it is selected and in `GROUP BY ALL`. Both tokens are + * grammar-gated before they reach SQL. */ export interface IAnalyticsMetricRequest { measures: string[]; dimensions?: string[]; filter?: MetricFilter; timeGrain?: string; + /** + * The single dimension that `timeGrain` buckets via `date_trunc`. Must be + * one of `dimensions` (so it is selected and in `GROUP BY ALL`) and is + * required whenever `timeGrain` is set. Grammar-gated as a SQL identifier. + */ + timeDimension?: string; limit?: number; format?: AnalyticsFormat; } From 544c2b77fc3f365452cc119c833ad93d83bac98a Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Thu, 9 Jul 2026 23:19:37 +0200 Subject: [PATCH 4/9] feat(appkit): metric OBO dispatch + cache-key isolation (PR2 phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit xavier loop: iteration 3 — Phase 3, completes PR2. Lane dispatch from the registration (metric-views.json executor), not a URL segment: OBO-lane metrics run on-behalf-of the requesting user via asUser(req) with a per-user cache scope; SP-lane metrics run as the app service principal with a shared scope. Executor + key computed inside a try so a missing/ whitespace OBO identity (from asUser/resolveUserId/deriveMetricExecutorKey) lands on the canonical 401 envelope, not an uncaught 500. composeMetricCacheKey builds metric:{key}:{argsHash}:{executorKey} over canonicalized args (sorted measures/dimensions, order-independent filter fingerprint via canonicalizeFilter, grain, timeDimension, limit). deriveMetricExecutorKey returns "sp" for SP and a sha256 of the trimmed user identity for OBO — the raw email/principal never enters the cache layer. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 57 ++- .../appkit/src/plugins/analytics/metric.ts | 137 +++++- .../plugins/analytics/tests/metric.test.ts | 407 +++++++++++++++++- 3 files changed, 592 insertions(+), 9 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index eceb4b9c..8cb12588 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -31,6 +31,8 @@ import { queryDefaults } from "./defaults"; import manifest from "./manifest.json"; import { buildMetricSql, + composeMetricCacheKey, + deriveMetricExecutorKey, loadMetricRegistry, validateMetricRequest, } from "./metric"; @@ -500,8 +502,11 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { * The `originalError` re-throw discipline preserves each error's structured * `errorCode`/`clientMessage` for the SSE error payload. * - * SP lane only in this phase — every registered metric runs as the service - * principal; OBO dispatch is wired in a later phase. + * Lane dispatch is driven by the registration: an SP-lane metric runs as the + * app service principal (shared cache); an OBO-lane metric runs + * on-behalf-of the requesting user via `asUser(req)` (per-user cache keyed by + * a hash of the user identity). The executor + key are computed inside a + * try so a missing/whitespace OBO identity lands on the canonical 401 path. */ async _handleMetricRoute( req: express.Request, @@ -575,14 +580,52 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - // SP lane only in this phase: the default executor + shared cache key. OBO - // dispatch (asUser(req) + per-user key) arrives in a later phase. - const executor = this; - const executorKey = "sp"; + // Lane dispatch. The lane comes from the registration (the entry's + // `executor` in metric-views.json), NOT a URL segment or `.obo.sql` + // filename: an OBO-lane metric runs on-behalf-of the requesting user + // (per-user cache via `asUser(req)`), an SP-lane metric as the app service + // principal (shared cache). Compute the executor + key INSIDE a try (as + // `_handleQueryRoute` does) so `asUser(req)`/`resolveUserId(req)` and + // `deriveMetricExecutorKey`'s missing-identity throw land on the canonical + // 401 envelope instead of surfacing as an uncaught 500 out of the stream. + let executor: AnalyticsPlugin; + let executorKey: string; + try { + const isObo = registration.lane === "obo"; + executor = isObo ? this.asUser(req) : this; + executorKey = deriveMetricExecutorKey({ + lane: registration.lane, + userIdentity: isObo ? this.resolveUserId(req) : undefined, + }); + } catch (err) { + if (err instanceof AppKitError) { + res.status(err.statusCode).json({ error: err.message, code: err.code }); + return; + } + throw err; + } + // Cache key. Composed over the canonicalized args (sorted measures/ + // dimensions, stable-sorted predicates, grain, timeDimension, limit) plus + // the `executorKey` — `"sp"` shares the cache across all users, a per-user + // identity hash isolates OBO callers. Delivery is JSON-only in v1 (the + // route always routes through `_executeJsonArrayPath`), so the key salts on + // a stable "JSON_ARRAY" constant rather than `request.format`: two calls + // that deliver identically must not fork the cache on an unused format + // field. const cacheConfig = { ...queryDefaults.cache, - cacheKey: ["analytics:metric", key, JSON.stringify(request), executorKey], + cacheKey: composeMetricCacheKey({ + metricKey: key, + measures: request.measures, + dimensions: request.dimensions, + timeGrain: request.timeGrain, + timeDimension: request.timeDimension, + filter: request.filter, + format: "JSON_ARRAY", + executorKey, + limit: request.limit, + }), }; // Cache/retry/timeout scoped to the SQL execution itself (inner `execute`) diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index bce144cd..b970fc97 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; @@ -7,7 +8,7 @@ import { z } from "zod"; // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; -import { ValidationError } from "../../errors"; +import { AuthenticationError, ValidationError } from "../../errors"; import { createLogger } from "../../logging/logger"; import type { IAnalyticsMetricRequest, @@ -1001,3 +1002,137 @@ function renderDimensionClause( } return dim; } + +/** + * Compose the cache key for a metric-view request. + * + * Reserved namespace `metric` separates metric-view caches from query caches. + * The returned array is consumed by `CacheManager.generateKey`, which + * concatenates and sha256-hashes the parts — the per-concern structure keeps + * the key inspectable in tests and debug logs without giving up determinism. + * + * Sort-before-hash collapses semantically equivalent calls onto one entry: + * - `measures` / `dimensions`: lexicographic sort. + * - `filter`: predicates inside each AND/OR group are stable-sorted by + * `(member, operator)` and the group kind (`and` vs `or`) is preserved by + * {@link canonicalizeFilter}; values are included verbatim so distinct + * filter values yield distinct keys. + * + * `timeGrain` AND `timeDimension` both salt the key: each independently + * changes the emitted SQL (the grain literal and which column is bucketed), so + * two calls that differ only in one of them must not share a cache entry. + * + * `executorKey` is `"sp"` for SP-lane entries (shared cache) or a sha256 hash + * of the end user's identity for OBO-lane entries (per-user isolation) — see + * {@link deriveMetricExecutorKey}. + */ +export function composeMetricCacheKey(input: { + metricKey: string; + measures: string[]; + dimensions?: string[]; + timeGrain?: string; + timeDimension?: string; + filter?: MetricFilter; + format: string; + executorKey: string; + limit?: number; +}): string[] { + const sortedMeasures = [...input.measures].sort(); + const sortedDimensions = [...(input.dimensions ?? [])].sort(); + const filterFingerprint = + input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + return [ + "metric", + input.metricKey, + input.format, + sortedMeasures.join(","), + sortedDimensions.join(","), + input.timeGrain ?? "_", + input.timeDimension ?? "_", + filterFingerprint, + typeof input.limit === "number" ? String(input.limit) : "_", + input.executorKey, + ]; +} + +/** + * Derive the cache executor key from a metric registration's lane and the + * caller's user identity. + * + * Returns `"sp"` for SP-lane entries (every caller shares the cache) and a + * sha256 hex digest of the user identity for OBO-lane entries (each user gets + * an isolated cache scope). + * + * The user identity is hashed — never stored verbatim — so the cache layer + * (which logs keys at debug level and persists them in any cache backend) + * never sees raw user emails or principal names. A stable, opaque token is + * exactly what we need: same user → same key (so cache hits work), different + * users → different keys (so isolation holds), and reverse lookup is + * infeasible. + * + * For OBO requests without a resolvable identity (missing or whitespace-only + * user id), throw `AuthenticationError.missingUserId()` rather than falling + * back to a shared `"anonymous"` sentinel — distinct misconfigured callers + * would otherwise collide into one cache scope and read each other's cached + * results. The route computes this inside its try/catch, so the throw lands on + * the canonical 401 envelope. + */ +export function deriveMetricExecutorKey(input: { + lane: MetricLane; + userIdentity?: string | null; +}): string { + if (input.lane === "sp") { + return "sp"; + } + // OBO lane — hash the user identity so the raw email/principal never reaches + // the cache layer. Missing/whitespace identity is a hard auth failure: the + // alternative ("anonymous" sentinel) collides every misconfigured caller + // into a single cache scope, so user A's results could leak to user B. + const identity = input.userIdentity?.trim(); + if (!identity) { + throw AuthenticationError.missingUserId(); + } + return createHash("sha256").update(identity).digest("hex"); +} + +/** + * Produce a deterministic string fingerprint of the filter tree. + * + * The fingerprint sorts predicates within each AND/OR group by + * `(member, operator)` and recursively canonicalizes nested groups. Values are + * included verbatim so cache entries differ when the filter targets different + * values (`region in [EMEA]` vs `region in [APAC]` → different keys; `equals A` + * vs `equals B` → different keys), while order-insensitive predicate lists + * collapse to the same key. + */ +function canonicalizeFilter(node: MetricFilter): string { + if (node === null || typeof node !== "object") { + return "_"; + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + return `${groupKey}()`; + } + + const sorted = sortFilterChildren(children); + const childFingerprints = sorted.map(canonicalizeFilter); + return `${groupKey}(${childFingerprints.join(",")})`; + } + + // Leaf predicate. Use JSON.stringify (not String) for the value segment so + // strings carrying the `|` separator cannot collide with split arrays — + // e.g. `["a", "b"]` and `["a|string:b"]` stay distinct fingerprints. + const p = node as MetricPredicate; + const valuesPart = p.values + ? p.values.map((v) => `${typeof v}:${JSON.stringify(v)}`).join("|") + : ""; + return `p(${p.member}/${p.operator}/${valuesPart})`; +} diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 50efd534..253be756 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -10,13 +10,20 @@ import { } from "@tools/test-helpers"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; import { ServiceContext } from "../../../context/service-context"; +import { AuthenticationError } from "../../../errors"; import { AnalyticsPlugin } from "../analytics"; import { buildMetricSql, + composeMetricCacheKey, + deriveMetricExecutorKey, loadMetricRegistry, validateMetricRequest, } from "../metric"; -import type { IAnalyticsConfig, MetricRegistration } from "../types"; +import type { + IAnalyticsConfig, + MetricFilter, + MetricRegistration, +} from "../types"; // Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s // cache interceptor is a no-op pass-through (each request re-executes). @@ -1263,3 +1270,401 @@ describe("metric — filter translator", () => { }); }); }); + +// ── Phase 3: cache-key composition. `composeMetricCacheKey` produces the +// array `CacheManager.generateKey` concatenates + sha256s; the invariants +// below are what make the cache both correct (semantically equal calls collapse) +// and safe (distinct args / executors never collide). +describe("composeMetricCacheKey", () => { + const base: { + metricKey: string; + measures: string[]; + format: string; + executorKey: string; + } = { + metricKey: "revenue", + measures: ["arr"], + format: "JSON_ARRAY", + executorKey: "sp", + }; + + test("measure ORDER does not affect the key (sorted before hashing)", () => { + const a = composeMetricCacheKey({ + ...base, + measures: ["arr", "revenue"], + }); + const b = composeMetricCacheKey({ + ...base, + measures: ["revenue", "arr"], + }); + expect(a).toEqual(b); + }); + + test("dimension ORDER does not affect the key (sorted before hashing)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["region", "segment"], + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["segment", "region"], + }); + expect(a).toEqual(b); + }); + + test("different measures → different keys", () => { + const a = composeMetricCacheKey({ ...base, measures: ["arr"] }); + const b = composeMetricCacheKey({ ...base, measures: ["mrr"] }); + expect(a).not.toEqual(b); + }); + + test("different dimensions → different keys", () => { + const a = composeMetricCacheKey({ ...base, dimensions: ["region"] }); + const b = composeMetricCacheKey({ ...base, dimensions: ["segment"] }); + expect(a).not.toEqual(b); + }); + + test("different timeGrain → different keys", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date", + }); + expect(a).not.toEqual(b); + }); + + test("different timeDimension → different keys (new field salts the key)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "ship_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "ship_date"], + timeGrain: "day", + timeDimension: "ship_date", + }); + expect(a).not.toEqual(b); + }); + + test("different limit → different keys", () => { + const a = composeMetricCacheKey({ ...base, limit: 10 }); + const b = composeMetricCacheKey({ ...base, limit: 20 }); + expect(a).not.toEqual(b); + }); + + test("predicate ORDER inside a group does not affect the key (canonicalizeFilter)", () => { + const a = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Ent"] }, + ], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "segment", operator: "equals", values: ["Ent"] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + } as MetricFilter, + }); + expect(a).toEqual(b); + }); + + test("distinct filter VALUES → distinct keys", () => { + const a = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "equals", + values: ["EMEA"], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "equals", + values: ["APAC"], + } as MetricFilter, + }); + expect(a).not.toEqual(b); + }); + + test("a filter present vs absent → different keys", () => { + const withFilter = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "set", + } as MetricFilter, + }); + const withoutFilter = composeMetricCacheKey({ ...base }); + expect(withFilter).not.toEqual(withoutFilter); + }); + + test("SP vs OBO (same args) → different keys via executorKey", () => { + const sp = composeMetricCacheKey({ ...base, executorKey: "sp" }); + const obo = composeMetricCacheKey({ + ...base, + executorKey: deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }), + }); + expect(sp).not.toEqual(obo); + }); +}); + +// ── Phase 3: executor-key isolation. The key is what scopes the cache — `"sp"` +// shares it across all callers, a per-user hash isolates OBO callers. The raw +// identity must never enter the key verbatim (privacy: cache keys are logged +// and persisted). +describe("deriveMetricExecutorKey", () => { + test("SP lane → literal 'sp' (shared cache)", () => { + expect(deriveMetricExecutorKey({ lane: "sp" })).toBe("sp"); + }); + + test("SP lane ignores any supplied identity", () => { + expect(deriveMetricExecutorKey({ lane: "sp", userIdentity: "alice" })).toBe( + "sp", + ); + }); + + test("OBO lane hashes the identity — raw identity never appears in the key", () => { + const key = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice@example.com", + }); + expect(key).toMatch(/^[a-f0-9]{64}$/); + expect(key).not.toContain("alice"); + }); + + test("OBO same identity → stable key (cache hits work)", () => { + const a = deriveMetricExecutorKey({ lane: "obo", userIdentity: "alice" }); + const b = deriveMetricExecutorKey({ lane: "obo", userIdentity: "alice" }); + expect(a).toBe(b); + }); + + test("OBO identity is trimmed before hashing (whitespace does not fork the scope)", () => { + const padded = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: " alice ", + }); + const bare = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }); + expect(padded).toBe(bare); + }); + + test("OBO different identities → different keys (isolation holds)", () => { + const alice = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }); + const bob = deriveMetricExecutorKey({ lane: "obo", userIdentity: "bob" }); + expect(alice).not.toBe(bob); + }); + + test("OBO empty identity → throws AuthenticationError (no shared 'anonymous' scope)", () => { + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: "" }), + ).toThrow(AuthenticationError); + }); + + test("OBO whitespace-only identity → throws AuthenticationError", () => { + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: " " }), + ).toThrow(AuthenticationError); + }); + + test("OBO missing (undefined/null) identity → throws AuthenticationError", () => { + expect(() => deriveMetricExecutorKey({ lane: "obo" })).toThrow( + AuthenticationError, + ); + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: null }), + ).toThrow(AuthenticationError); + }); +}); + +// ── Phase 3: lane dispatch at the handler level. The lane comes from the +// registration (the entry's `executor` in metric-views.json), NOT the URL: +// OBO-lane routes through `asUser(req)`, SP-lane through the default executor. +// A missing/whitespace OBO identity must land on the canonical 401 envelope, +// never an out-of-envelope 500. +describe("metric route — lane dispatch (Phase 3)", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + test("OBO-lane registration routes through asUser(req)", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }); + + const asUserSpy = vi.spyOn(plugin, "asUser"); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": "alice", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The OBO lane dispatched through the user-scoped executor… + expect(asUserSpy).toHaveBeenCalledWith(mockReq); + // …and the SQL still reached the warehouse and streamed a result. + expect(executeMock).toHaveBeenCalled(); + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + }); + + test("SP-lane registration uses the default executor (asUser not called)", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const asUserSpy = vi.spyOn(plugin, "asUser"); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + // Even with user headers present, an SP-lane metric must NOT impersonate. + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": "alice", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(asUserSpy).not.toHaveBeenCalled(); + expect(executeMock).toHaveBeenCalled(); + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + }); + + test("OBO metric with no user identity → canonical 401, no SQL executed", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + // No x-forwarded-* headers at all → the OBO dispatch throws an + // AuthenticationError, caught and returned as a canonical 401 envelope. + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(401); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ code: "AUTHENTICATION_ERROR" }), + ); + // Landed on the 401 path before any SQL and without opening the SSE stream. + expect(executeMock).not.toHaveBeenCalled(); + expect(mockRes.write).not.toHaveBeenCalled(); + }); + + test("OBO metric with whitespace-only user identity → canonical 401", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": " ", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(401); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ code: "AUTHENTICATION_ERROR" }), + ); + expect(executeMock).not.toHaveBeenCalled(); + }); +}); From 13afba65a1049ec231c527c95f9a6081ee01a558 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 10:32:27 +0200 Subject: [PATCH 5/9] fix(appkit): harden metric route lookup, arg uniqueness, sort key, format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defensive fixes to the PR2 metric-view runtime surfaced by adversarial review: - Registry lookup (B): build the registry with a null prototype and gate the route read with Object.hasOwn, so a metric key colliding with an inherited Object.prototype member (__proto__, constructor, toString, …) can no longer resolve to a truthy non-registration and bypass the unknown-key 404. - Measure/dimension uniqueness (C): reject a name that repeats within measures, within dimensions, or across both. Measures and dimensions alias to their own name in the SELECT list (MEASURE(x) AS x, x), so a duplicate collapses to one row-object key and silently drops a value during row materialization. - Filter sort key (D): delimit the (member, operator) sort key with "/" instead of a bare concatenation, so two distinct pairs cannot map to the same key and fork the cache on semantically equal filters. Matches the delimiter canonicalizeFilter already uses for its leaf fingerprint. - Format (F): reject any format other than JSON_ARRAY (legacy aliases normalized first). The metric route delivers JSON rows only at v1; an Arrow request previously returned JSON silently instead of failing loud. Adds 10 tests covering all four. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 10 +- .../appkit/src/plugins/analytics/metric.ts | 61 +++++++- .../plugins/analytics/tests/metric.test.ts | 146 ++++++++++++++++++ 3 files changed, 214 insertions(+), 3 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 8cb12588..29eea810 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -544,7 +544,15 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - const registration = registry[key]; + // Own-property lookup: never resolve `key` to an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …). + // The loader already builds a null-prototype registry, but the load-error + // fallback and test-injected registries are plain objects, so gate the + // read here too — an inherited hit would otherwise bypass the 404 below + // and flow a non-registration value into execution. + const registration = Object.hasOwn(registry, key) + ? registry[key] + : undefined; if (!registration) { // Don't echo the user-supplied `key` back in the public response — // confirming "metric X is not registered" lets a probe enumerate keys by diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index b970fc97..519d2659 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -18,6 +18,7 @@ import type { MetricPredicate, MetricRegistration, } from "./types"; +import { normalizeAnalyticsFormat } from "./types"; const logger = createLogger("analytics:metric"); @@ -202,7 +203,13 @@ export function loadMetricRegistry( throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); } - const registry: Record = {}; + // Null-prototype map so a metric key that collides with an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) + // cannot resolve to a truthy non-registration at the `registry[key]` read + // site and slip past the unknown-key 404. Keys are still grammar-gated by + // `metricKeySchema` (identifier shape), but the null prototype removes the + // whole class of inherited-property lookups as a boundary. + const registry: Record = Object.create(null); for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { registry[key] = { key, @@ -344,6 +351,48 @@ const metricRequestSchema = z validateFilterTree(value.filter, ctx, ["filter"], 0); } + // Uniqueness. Measures and dimensions become SELECT columns aliased to + // their own name (`MEASURE(x) AS x`, `x`); the warehouse returns one row + // object keyed by column name, so a repeated name — a duplicate measure, a + // duplicate dimension, or a name appearing as BOTH a measure and a + // dimension — collapses to a single key and silently drops a value during + // row materialization. Reject the collision here rather than emit SQL that + // corrupts rows. (The grammar gate at build time is a safety boundary, not + // a uniqueness check, so this lives in validation.) + const seen = new Set(); + const collided = new Set(); + for (const name of [...value.measures, ...(value.dimensions ?? [])]) { + if (seen.has(name)) { + collided.add(name); + } + seen.add(name); + } + if (collided.size > 0) { + ctx.addIssue({ + code: "custom", + message: + "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", + path: ["measures"], + }); + } + + // Delivery format. The metric route delivers JSON rows only at v1 (the + // cache salts on a fixed "JSON_ARRAY" and the route always routes through + // the JSON path). Accepting an Arrow format would silently return JSON — + // reject it loudly until Arrow parity ships. Legacy aliases normalize + // first (`JSON`→`JSON_ARRAY`, `ARROW`→`ARROW_STREAM`). + if ( + value.format != null && + normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" + ) { + ctx.addIssue({ + code: "custom", + message: + "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", + path: ["format"], + }); + } + // Cross-field grain rules. `timeGrain` buckets exactly one selected // dimension via `date_trunc`, so it requires an explicit `timeDimension`, // and that dimension must be selected (so it is in the SELECT list and @@ -824,7 +873,15 @@ function sortFilterChildren( !("or" in child) ) { const p = child as MetricPredicate; - key = `${p.member}${p.operator}`; + // Separate the fields with "/" so the (member, operator) pair maps to + // the sort key injectively. A delimiter-free concatenation collides — + // member "ab" + op "c" and member "a" + op "bc" both → "abc" — which + // tie-breaks two distinct pairs to input order and forks the cache key + // on semantically equal filters. "/" can never appear in either field + // (members match DIMENSION_NAME_PATTERN, operators are the alpha-only + // enum), matching the leaf-fingerprint delimiter canonicalizeFilter + // already uses. + key = `${p.member}/${p.operator}`; isPredicate = true; } else { // Nested groups don't have a single (member, operator) — keep their diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 253be756..e339be23 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -466,6 +466,43 @@ describe("analytics metric route (Phase 1)", () => { expect(mockRes.json).toHaveBeenCalledWith({ error: "Metric not found" }); }); + test.each(["__proto__", "constructor", "toString", "hasOwnProperty"])( + "inherited Object.prototype key %j → 404, no execution (own-property lookup)", + async (dangerousKey) => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + // A real (own) entry so the registry is populated but does NOT contain + // the dangerous key. Without the own-property guard, `registry[key]` + // would resolve the inherited prototype member and slip past the 404. + setRegistry(plugin, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: dangerousKey }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(mockRes.json).toHaveBeenCalledWith({ + error: "Metric not found", + }); + expect(executeMock).not.toHaveBeenCalled(); + }, + ); + test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { const plugin = new AnalyticsPlugin(config); const { router, getHandler } = createMockRouter(); @@ -554,6 +591,22 @@ describe("loadMetricRegistry", () => { }); }); + test("registry has a null prototype (no inherited-property lookups)", () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + + const registry = loadMetricRegistry(dir); + expect(Object.getPrototypeOf(registry)).toBeNull(); + // A key that would resolve to a truthy inherited member on a plain object + // resolves to undefined here. + expect((registry as Record).toString).toBeUndefined(); + expect((registry as Record).__proto__).toBeUndefined(); + }); + test("malformed JSON throws", () => { writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); expect(() => loadMetricRegistry(dir)).toThrow(/Failed to parse/); @@ -1092,6 +1145,73 @@ describe("metric — filter translator", () => { }); }); + describe("validator — measure/dimension uniqueness", () => { + // Measures and dimensions become SELECT columns aliased to their own name; + // a repeated name collapses to a single row-object key and silently drops + // a value during row materialization, so uniqueness is enforced up front. + test("rejects a duplicate measure", () => { + expect(() => + validateMetricRequest({ measures: ["arr", "arr"] }), + ).toThrowError(/fields:.*measures/); + }); + + test("rejects a duplicate dimension", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region", "region"], + }), + ).toThrowError(/fields:.*measures/); + }); + + test("rejects a name that is BOTH a measure and a dimension", () => { + // The corruption case: `SELECT MEASURE(x) AS x, x ... GROUP BY ALL` + // materializes two `x` columns and the second overwrites the first. + expect(() => + validateMetricRequest({ + measures: ["revenue"], + dimensions: ["revenue"], + }), + ).toThrowError(/fields:.*measures/); + }); + + test("accepts distinct measures and dimensions (no false positive)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr", "mrr"], + dimensions: ["region", "segment"], + }), + ).not.toThrow(); + }); + }); + + describe("validator — format (JSON-only at v1)", () => { + test("accepts JSON_ARRAY", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "JSON_ARRAY" }), + ).not.toThrow(); + }); + + test("accepts the legacy JSON alias (normalizes to JSON_ARRAY)", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "JSON" }), + ).not.toThrow(); + }); + + test("rejects ARROW_STREAM (not implemented on the metric route)", () => { + // Fail loud rather than silently deliver JSON for an Arrow request. + expect(() => + validateMetricRequest({ measures: ["arr"], format: "ARROW_STREAM" }), + ).toThrowError(/fields:.*format/); + }); + + test("rejects the legacy ARROW alias (normalizes to ARROW_STREAM)", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "ARROW" }), + ).toThrowError(/fields:.*format/); + }); + }); + describe("timeGrain + timeDimension (validator)", () => { test("a grammar-valid timeGrain + timeDimension (in dimensions) is accepted", () => { expect(() => @@ -1384,6 +1504,32 @@ describe("composeMetricCacheKey", () => { expect(a).toEqual(b); }); + test("predicate ORDER is stable for two ops on the SAME member (sort-key delimiter)", () => { + // Same member, different operators — the sort key is `${member}/${operator}`, + // so the "/" delimiter (not a raw concatenation) is what disambiguates the + // pair and keeps the ordering — and thus the key — independent of input + // order. Two range bounds on one column is the everyday shape of this. + const a = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "revenue", operator: "gt", values: [100] }, + { member: "revenue", operator: "lt", values: [200] }, + ], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "revenue", operator: "lt", values: [200] }, + { member: "revenue", operator: "gt", values: [100] }, + ], + } as MetricFilter, + }); + expect(a).toEqual(b); + }); + test("distinct filter VALUES → distinct keys", () => { const a = composeMetricCacheKey({ ...base, From fd232089a70eb2b7d2ac624405da165e29acf83f Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 10:48:32 +0200 Subject: [PATCH 6/9] fix(appkit): unify metric-view FQN grammar + quote at SQL interpolation The runtime, the shared schema, and the type-generator disagreed on what a metric-view `source` FQN may contain. The shared schema and typegen accept the full UC quoted-identifier grammar (hyphens, non-ASCII) and typegen backtick- quotes before interpolation; the runtime used a narrower ASCII allowlist (assertSafeFqn) AND interpolated the FQN UNQUOTED. So a documented-legal UC name like `prod-data.analytics.revenue` passed config + type generation but threw (or, if the pattern were widened, emitted invalid SQL) at request time. Converge on one grammar + one escaper: - Move `isValidFqn` (three-part UC predicate) and `quoteFqnForSql` (backtick escaper) into the shared zod-free leaf `metric-fqn.ts`, beside `UC_FQN_PATTERN`. Grammar and quoting now have a single home both the type-generator and the analytics runtime import. `describe.ts` and `config.ts` import them back; `mv-registry.test.ts` imports the escaper from the leaf. - Runtime `buildMetricSql` replaces the narrow `assertSafeFqn` regex with `quoteSafeFqn`: validate via `isValidFqn`, then interpolate the `quoteFqnForSql`-escaped FQN. Quoting is the injection boundary, so the runtime accepts exactly what UC (and the schema, and typegen) accept. - `UC_THREE_PART_FQN_PATTERN` stays in `metric-source.ts` so zod still emits a JSON-schema `pattern`; it derives from the same per-segment charset as `isValidFqn`, so the two shapes cannot diverge. Also folds in cache-key hardening (in-flight): salt the metric cache key with `source` so repointing a metric key to a different FQN cannot stale-serve, and only salt `timeDimension` when `timeGrain` is set (it has no SQL effect otherwise). `renderDimensionClause` re-gates the grain against TIME_GRAIN_PATTERN at its interpolation point. Tests: requote the ~13 emitted-FROM assertions; add regression tests that a hyphenated FQN is accepted+quoted and that a backtick-bearing segment is neutralized by doubling. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 1 + .../appkit/src/plugins/analytics/metric.ts | 90 +++++++++----- .../plugins/analytics/tests/metric.test.ts | 111 ++++++++++++++++-- .../src/type-generator/mv-registry/config.ts | 27 ++--- .../type-generator/mv-registry/describe.ts | 51 ++------ .../type-generator/tests/mv-registry.test.ts | 5 +- packages/shared/src/schemas/metric-fqn.ts | 77 ++++++++++++ 7 files changed, 256 insertions(+), 106 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 29eea810..c8bf5d4e 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -625,6 +625,7 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ...queryDefaults.cache, cacheKey: composeMetricCacheKey({ metricKey: key, + source: registration.source, measures: request.measures, dimensions: request.dimensions, timeGrain: request.timeGrain, diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index 519d2659..5ed31946 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -7,6 +7,10 @@ import { z } from "zod"; // `metric-views.json`. Imported from the shared source directly (matching the // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. +import { + isValidFqn, + quoteFqnForSql, +} from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { AuthenticationError, ValidationError } from "../../errors"; import { createLogger } from "../../logging/logger"; @@ -30,16 +34,6 @@ const logger = createLogger("analytics:metric"); const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); const METRIC_CONFIG_FILE = "metric-views.json"; -/** - * Three-part UC FQN matcher. The registry is parsed against the landed - * `metricSourceSchema`, which already validates `source` via the composed - * `UC_THREE_PART_FQN_PATTERN`; this is a belt-and-suspenders runtime fence for - * any code path that constructs SQL from a `source` outside that parse (the FQN - * cannot be parameterized — it is interpolated into the SQL string). - */ -const FQN_PATTERN = - /^[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*\.[a-zA-Z0-9_][a-zA-Z0-9_-]*$/; - /** * Validate measure names before they are interpolated into `MEASURE()`. * @@ -226,19 +220,35 @@ export function loadMetricRegistry( } /** - * SQL identifier safety guard — the FQN ships in the SQL string (it cannot be - * parameterized) so we re-check the regex at construction time. + * Validate a metric-view FQN and return it backtick-quoted for interpolation. + * + * The FQN ships in the SQL string — it cannot be parameterized — so it passes + * two shared, zod-free gates at construction time: + * + * 1. {@link isValidFqn} — the three-part UC grammar (exactly three segments, + * each matching `UC_FQN_PATTERN`). This is the SAME predicate the + * type-generator's describe seam uses and is derived from the same + * per-segment charset as the canonical Zod schema, so config-time, + * generation-time, and runtime accept exactly the same names. + * 2. {@link quoteFqnForSql} — backtick-quotes each segment (doubling embedded + * backticks) so the FQN cannot break out of its identifier position. This + * is the injection boundary; it is why the runtime can accept the full UC + * quoted-identifier grammar (hyphens, non-ASCII) rather than the narrow + * ASCII allowlist it used before. * - * The registry loader already enforces the FQN grammar via `metricSourceSchema`; - * this is a runtime fence for any future code path that constructs SQL outside - * of a parsed registry. + * The registry loader already enforces the grammar via `metricSourceSchema`; + * this re-gate is defense-in-depth for any code path that reaches + * `buildMetricSql` without going through a parsed registry (it is exported), + * matching how measures, dimensions, and the grain are each re-gated at their + * interpolation points. */ -function assertSafeFqn(fqn: string): void { - if (!FQN_PATTERN.test(fqn)) { +function quoteSafeFqn(fqn: string): string { + if (!isValidFqn(fqn)) { throw new Error( `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, ); } + return quoteFqnForSql(fqn); } /** @@ -658,9 +668,10 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { * - Every measure and dimension is gated by {@link MEASURE_NAME_PATTERN} / * {@link DIMENSION_NAME_PATTERN} before it is interpolated (column * references cannot be parameterized — they are SQL identifiers), and the - * FQN is re-checked by {@link assertSafeFqn}. There is deliberately NO name - * allowlist — the grammar gate is the security boundary. No user-supplied - * string reaches the SQL string without passing a grammar gate. + * FQN is validated and backtick-quoted by {@link quoteSafeFqn}. There is + * deliberately NO name allowlist — the grammar gate (plus quoting for the + * FQN) is the security boundary. No user-supplied string reaches the SQL + * string without passing a grammar gate. * - `GROUP BY ALL` is added when at least one dimension is requested. UC * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; * `GROUP BY ALL` is the documented form that works without re-listing each @@ -683,7 +694,7 @@ export function buildMetricSql( statement: string; parameters: Record; } { - assertSafeFqn(registration.source); + const quotedSource = quoteSafeFqn(registration.source); if (request.measures.length === 0) { throw new Error("buildMetricSql requires at least one measure."); @@ -743,7 +754,7 @@ export function buildMetricSql( } } - const statement = `SELECT ${selectList} FROM ${registration.source}${whereClause}${groupByClause}${limitClause}`; + const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; return { statement, parameters }; } @@ -1035,8 +1046,12 @@ function bindLikeValue( * grammar-gated before they reach the SQL string: * * - The grain is single-quoted (it is a `date_trunc` unit literal, NOT a bind - * param) and `TIME_GRAIN_PATTERN` upstream is the control that keeps a - * hostile token out of that quoted position. + * param), so it is re-checked against {@link TIME_GRAIN_PATTERN} here before + * interpolation. The schema also gates it, but `buildMetricSql` is exported + * and may be reached on a path that bypasses `validateMetricRequest`, so the + * grammar gate is applied at the interpolation point too — matching how + * every other interpolated identifier (measures, dimensions, `timeDimension`, + * the FQN) is re-gated in the builder. * - The column cannot be parameterized (it is an identifier), so it is * re-checked against {@link DIMENSION_NAME_PATTERN} here as belt-and- * suspenders even though the schema already gated `timeDimension`. @@ -1055,6 +1070,11 @@ function renderDimensionClause( `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, ); } + if (!TIME_GRAIN_PATTERN.test(timeGrain)) { + throw new Error( + `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, + ); + } return `date_trunc('${timeGrain}', ${dim}) AS ${dim}`; } return dim; @@ -1075,9 +1095,16 @@ function renderDimensionClause( * {@link canonicalizeFilter}; values are included verbatim so distinct * filter values yield distinct keys. * - * `timeGrain` AND `timeDimension` both salt the key: each independently - * changes the emitted SQL (the grain literal and which column is bucketed), so - * two calls that differ only in one of them must not share a cache entry. + * `source` (the metric view's UC FQN) salts the key so that repointing a + * metric `key` to a different FQN in `metric-views.json` cannot serve rows + * cached under the old source within the TTL — `metricKey` alone is not enough + * because the key→source binding is mutable config. + * + * `timeGrain` salts the key whenever set. `timeDimension` only salts the key + * when `timeGrain` is also set, because `renderDimensionClause` only applies + * `date_trunc('', )` when a grain is present — with no + * grain the field has no effect on the emitted SQL, so including it would fork + * the cache on an input that produced identical SQL. * * `executorKey` is `"sp"` for SP-lane entries (shared cache) or a sha256 hash * of the end user's identity for OBO-lane entries (per-user isolation) — see @@ -1085,6 +1112,7 @@ function renderDimensionClause( */ export function composeMetricCacheKey(input: { metricKey: string; + source: string; measures: string[]; dimensions?: string[]; timeGrain?: string; @@ -1098,14 +1126,20 @@ export function composeMetricCacheKey(input: { const sortedDimensions = [...(input.dimensions ?? [])].sort(); const filterFingerprint = input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + // `timeDimension` only changes the SQL when `timeGrain` is set (see + // renderDimensionClause); keying on it otherwise would fork the cache on a + // no-op field. + const timeDimensionPart = + input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; return [ "metric", input.metricKey, + input.source, input.format, sortedMeasures.join(","), sortedDimensions.join(","), input.timeGrain ?? "_", - input.timeDimension ?? "_", + timeDimensionPart, filterFingerprint, typeof input.limit === "number" ? String(input.limit) : "_", input.executorKey, diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index e339be23..cde55285 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -137,6 +137,36 @@ describe("analytics metric route (Phase 1)", () => { ), ).toThrow(/not a valid three-part UC FQN/); }); + + test("accepts a hyphenated UC-legal FQN and backtick-quotes it", () => { + // Regression for the grammar divergence: `prod-data` is a legal UC + // object name (hyphens are allowed in quoted identifiers) and passes the + // shared schema + typegen, but the old narrow runtime pattern + // [a-zA-Z0-9_-] anchored per segment REJECTED it — a latent prod break on + // a documented-legal name. The builder now accepts it and quotes every + // segment, so it reaches the warehouse as valid SQL rather than throwing. + const { statement } = buildMetricSql( + { key: "x", source: "prod-data.analytics.revenue", lane: "sp" }, + { measures: ["arr"] }, + ); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM `prod-data`.`analytics`.`revenue`", + ); + }); + + test("backtick-quoting neutralizes an injection-shaped FQN segment", () => { + // A source carrying a backtick would break out of the quoted identifier + // if interpolated raw; quoteFqnForSql doubles it. (isValidFqn rejects + // most such names first, but the quoting is the actual injection + // boundary — this asserts it, not just the grammar gate.) + const { statement } = buildMetricSql( + { key: "x", source: "cat.sch.re`v", lane: "sp" }, + { measures: ["arr"] }, + ); + expect(statement).toBe( + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`re``v`", + ); + }); }); // ── Measures-only SQL shape. @@ -152,7 +182,7 @@ describe("analytics metric route (Phase 1)", () => { measures: ["arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", ); expect(parameters).toEqual({}); }); @@ -162,7 +192,7 @@ describe("analytics metric route (Phase 1)", () => { measures: ["revenue", "arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM cat.sch.revenue_metrics", + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -172,7 +202,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 10.9, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics LIMIT 10", + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", ); }); }); @@ -192,7 +222,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -202,7 +232,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["segment", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region, segment FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, region, segment FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -212,7 +242,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -222,7 +252,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: [], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -233,7 +263,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 100, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM cat.sch.revenue_metrics GROUP BY ALL LIMIT 100", + "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", ); }); @@ -243,7 +273,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["order_date", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).not.toContain("date_trunc"); }); @@ -267,7 +297,7 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).toContain( "date_trunc('month', order_date) AS order_date", @@ -283,9 +313,24 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM cat.sch.revenue_metrics GROUP BY ALL", + "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); + + test("a grammar-invalid timeGrain throws in the builder (defense-in-depth, even if validation is bypassed)", () => { + // buildMetricSql is exported and may be reached on a path that skips + // validateMetricRequest, so the grain — interpolated into a single-quoted + // date_trunc literal — is re-gated at the interpolation point. A quote- + // breakout payload must be refused by the builder itself. + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month'); DROP TABLE t;--", + timeDimension: "order_date", + }), + ).toThrow(/not a valid grain token/); + }); }); // ── Phase 2: dimension grammar gate. A dimension failing @@ -349,7 +394,8 @@ describe("analytics metric route (Phase 1)", () => { expect(executeMock).toHaveBeenCalledWith( expect.anything(), expect.objectContaining({ - statement: "SELECT MEASURE(arr) AS arr FROM cat.sch.revenue_metrics", + statement: + "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", warehouse_id: "test-warehouse-id", }), expect.any(AbortSignal), @@ -1369,7 +1415,7 @@ describe("metric — filter translator", () => { expect(executeMock.mock.calls[0][1]).toEqual( expect.objectContaining({ statement: - "SELECT MEASURE(ghost_measure) AS ghost_measure FROM cat.sch.revenue_metrics", + "SELECT MEASURE(ghost_measure) AS ghost_measure FROM `cat`.`sch`.`revenue_metrics`", }), ); @@ -1398,16 +1444,55 @@ describe("metric — filter translator", () => { describe("composeMetricCacheKey", () => { const base: { metricKey: string; + source: string; measures: string[]; format: string; executorKey: string; } = { metricKey: "revenue", + source: "cat.sch.revenue_metrics", measures: ["arr"], format: "JSON_ARRAY", executorKey: "sp", }; + test("same key but different source → different keys (config repoint is not stale-served)", () => { + const a = composeMetricCacheKey({ ...base, source: "cat.sch.old_view" }); + const b = composeMetricCacheKey({ ...base, source: "cat.sch.new_view" }); + expect(a).not.toEqual(b); + }); + + test("timeDimension does NOT fork the key when timeGrain is absent (no SQL effect)", () => { + // Without a grain, renderDimensionClause emits the bare column, so + // timeDimension has no effect on the SQL — two such calls must cache-hit. + const withTd = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeDimension: "order_date", + }); + const withoutTd = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + }); + expect(withTd).toEqual(withoutTd); + }); + + test("timeDimension DOES fork the key when timeGrain is set (changes the SQL)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "region", + }); + expect(a).not.toEqual(b); + }); + test("measure ORDER does not affect the key (sorted before hashing)", () => { const a = composeMetricCacheKey({ ...base, diff --git a/packages/appkit/src/type-generator/mv-registry/config.ts b/packages/appkit/src/type-generator/mv-registry/config.ts index 1995f072..e91e3b95 100644 --- a/packages/appkit/src/type-generator/mv-registry/config.ts +++ b/packages/appkit/src/type-generator/mv-registry/config.ts @@ -94,26 +94,13 @@ function isValidMetricKey(key: string): boolean { return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key); } -/** - * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? - * - * Well-formed = exactly three non-empty, dot-separated segments, each a valid - * Unity Catalog object name per the shared {@link UC_FQN_PATTERN} (the single - * source of truth, also used by the canonical Zod schema). Used as the - * defense-in-depth re-check at the describe fetcher seam; {@link resolveMetricConfig} - * runs the same checks but with specific, staged error messages. - * - * @note Segment length ({@link MAX_FQN_SEGMENT_LENGTH}) is NOT checked here — - * an over-long but otherwise legal name is still "valid shape". The length cap - * is a separate concern enforced (with its own message) in resolveMetricConfig. - */ -export function isValidFqn(fqn: string): boolean { - const segments = fqn.split("."); - if (segments.length !== FQN_SEGMENT_COUNT) { - return false; - } - return segments.every((segment) => UC_FQN_PATTERN.test(segment)); -} +// `isValidFqn` (the three-part UC FQN predicate) now lives in the shared +// zod-free leaf alongside `UC_FQN_PATTERN` and `quoteFqnForSql`, so the +// type-generator and the analytics runtime share one grammar + one escaper. +// `resolveMetricConfig` below intentionally does NOT call it — it re-derives +// the same checks inline so it can emit specific, staged error messages +// (arity, per-segment charset, per-segment length) rather than a single +// boolean. /** * Field allowlists enforced by {@link resolveMetricConfig}. diff --git a/packages/appkit/src/type-generator/mv-registry/describe.ts b/packages/appkit/src/type-generator/mv-registry/describe.ts index db3231d6..a77caf75 100644 --- a/packages/appkit/src/type-generator/mv-registry/describe.ts +++ b/packages/appkit/src/type-generator/mv-registry/describe.ts @@ -1,7 +1,13 @@ import type { WorkspaceClient } from "@databricks/sdk-experimental"; +// Grammar + SQL-quoting for metric-view FQNs live together in the shared, +// zod-free leaf so the type-generator and the analytics runtime validate and +// escape against one source of truth (see the module doc in metric-fqn.ts). +import { + isValidFqn, + quoteFqnForSql, +} from "../../../../shared/src/schemas/metric-fqn"; import { type DescribeFormatMemo, describeAdaptive } from "../statement-result"; import type { DatabricksStatementExecutionResponse } from "../types"; -import { isValidFqn } from "./config"; import type { DescribeFetcher, MetricColumnMetadata } from "./types"; /** @@ -284,49 +290,6 @@ function inferTimeGrains(type: string): string[] | undefined { return undefined; } -/** - * Quote a dot-separated FQN for safe interpolation into a Spark/Databricks SQL - * statement. - * - * Each dot-split segment is wrapped in backtick-quoted-identifier syntax. The - * one character that can break out of a backtick-quoted identifier is the - * backtick itself, escaped by doubling (`` ` `` → `` `` ``) — so every backtick - * inside a segment is doubled before the segment is wrapped. Control characters - * and newlines have no valid escape inside a quoted identifier, so a segment - * containing one is rejected outright. - * - * This is a pure, standalone escaper: it is intentionally independent of FQN - * naming validation ({@link isValidFqn}). Naming validation decides whether an - * FQN is an acceptable metric source; this function only guarantees that - * whatever it is handed cannot break out of the quoted identifier it produces. - * - * An ordinary identifier is unchanged apart from the wrapping backticks: - * `catalog.schema.view` → `` `catalog`.`schema`.`view` ``. - * - * @param fqn - Dot-separated identifier (e.g. `catalog.schema.view`). - * @returns The backtick-quoted, escaped identifier ready for interpolation. - * @throws If any segment contains a control character or newline. - */ -export function quoteFqnForSql(fqn: string): string { - // Reject anything that cannot be represented inside a backtick-quoted - // identifier. \p{Cc} is the Unicode "control" category, which covers C0 - // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. - const CONTROL_OR_NEWLINE = /\p{Cc}/u; - return fqn - .split(".") - .map((segment) => { - if (CONTROL_OR_NEWLINE.test(segment)) { - throw new Error( - `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, - ); - } - // Double every backtick — the only break-out from a backtick-quoted - // identifier — then wrap the whole segment in backticks. - return `\`${segment.replace(/`/g, "``")}\``; - }) - .join("."); -} - /** * Build a DescribeFetcher from a real WorkspaceClient + warehouseId. */ diff --git a/packages/appkit/src/type-generator/tests/mv-registry.test.ts b/packages/appkit/src/type-generator/tests/mv-registry.test.ts index 06886140..8a06aca6 100644 --- a/packages/appkit/src/type-generator/tests/mv-registry.test.ts +++ b/packages/appkit/src/type-generator/tests/mv-registry.test.ts @@ -3,13 +3,16 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; +// `quoteFqnForSql` now lives in the shared zod-free leaf alongside the FQN +// grammar (moved so the analytics runtime can reuse it); the describe seam +// imports it from there. +import { quoteFqnForSql } from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { readMetricConfig, resolveMetricConfig } from "../mv-registry/config"; import { createWorkspaceDescribeFetcher, extractMetricColumns, parseDescribeTableExtendedJson, - quoteFqnForSql, } from "../mv-registry/describe"; import { generateMetricTypeDeclarations } from "../mv-registry/render-types"; import { syncMetrics } from "../mv-registry/sync"; diff --git a/packages/shared/src/schemas/metric-fqn.ts b/packages/shared/src/schemas/metric-fqn.ts index b958cd1a..98fca3d5 100644 --- a/packages/shared/src/schemas/metric-fqn.ts +++ b/packages/shared/src/schemas/metric-fqn.ts @@ -81,3 +81,80 @@ export const MAX_UC_OBJECT_NAME_LENGTH = 255; */ // biome-ignore lint/suspicious/noControlCharactersInRegex: UC explicitly prohibits ASCII control characters in object names; this negated class encodes that rule. export const UC_FQN_PATTERN = /^[^\x00-\x20\x7f./]+$/; + +/** A metric view FQN is exactly three segments: catalog.schema.metric_view. */ +const FQN_SEGMENT_COUNT = 3; + +/** + * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? + * + * Well-formed = exactly three non-empty, dot-separated segments, each a valid + * Unity Catalog object name per {@link UC_FQN_PATTERN}. This is the shared, + * zod-free grammar check reused by every layer that must agree on FQN shape: + * the type-generator's config resolver and describe seam, and the analytics + * runtime's SQL builder. It is the boolean sibling of the composed + * `UC_THREE_PART_FQN_PATTERN` regex in `./metric-source.ts` (which stays a + * regex so zod can emit a JSON-schema `pattern`); both derive their per-segment + * charset from {@link UC_FQN_PATTERN}, so they cannot diverge. + * + * @note Segment length ({@link MAX_UC_OBJECT_NAME_LENGTH}) is NOT checked here — + * an over-long but otherwise legal name is still "valid shape". Callers that + * care about the length cap enforce it separately with their own message. + * + * @example + * isValidFqn("main.analytics.revenue"); // true + * isValidFqn("prod-data.analytics.rev"); // true (hyphens are UC-legal) + * isValidFqn("main.analytics"); // false (only two segments) + */ +export function isValidFqn(fqn: string): boolean { + const segments = fqn.split("."); + if (segments.length !== FQN_SEGMENT_COUNT) { + return false; + } + return segments.every((segment) => UC_FQN_PATTERN.test(segment)); +} + +/** + * Quote a dot-separated FQN for safe interpolation into a Spark/Databricks SQL + * statement. + * + * Each dot-split segment is wrapped in backtick-quoted-identifier syntax. The + * one character that can break out of a backtick-quoted identifier is the + * backtick itself, escaped by doubling (`` ` `` → `` `` ``) — so every backtick + * inside a segment is doubled before the segment is wrapped. Control characters + * and newlines have no valid escape inside a quoted identifier, so a segment + * containing one is rejected outright. + * + * This is a pure, standalone escaper: it is intentionally independent of FQN + * naming validation ({@link isValidFqn}). Naming validation decides whether an + * FQN is an acceptable metric source; this function only guarantees that + * whatever it is handed cannot break out of the quoted identifier it produces. + * Grammar and quoting live together here so a metric source is validated and + * escaped against one shared source of truth. + * + * An ordinary identifier is unchanged apart from the wrapping backticks: + * `catalog.schema.view` → `` `catalog`.`schema`.`view` ``. + * + * @param fqn - Dot-separated identifier (e.g. `catalog.schema.view`). + * @returns The backtick-quoted, escaped identifier ready for interpolation. + * @throws If any segment contains a control character or newline. + */ +export function quoteFqnForSql(fqn: string): string { + // Reject anything that cannot be represented inside a backtick-quoted + // identifier. \p{Cc} is the Unicode "control" category, which covers C0 + // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. + const CONTROL_OR_NEWLINE = /\p{Cc}/u; + return fqn + .split(".") + .map((segment) => { + if (CONTROL_OR_NEWLINE.test(segment)) { + throw new Error( + `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, + ); + } + // Double every backtick — the only break-out from a backtick-quoted + // identifier — then wrap the whole segment in backticks. + return `\`${segment.replace(/`/g, "``")}\``; + }) + .join("."); +} From a1080f9378ab1df7fb2ff289799b6279e4d09691 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 11:11:32 +0200 Subject: [PATCH 7/9] fix(appkit): load metric registry lazily with an mtime-validated cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The metric registry memoized both success AND failure on the first `/metric/:key` request and never re-read: editing `metric-views.json` needed a server restart, and a transient first-request error latched a 503 forever — unlike the sibling `.sql` query path, which re-reads `config/queries/` per request. It was also a synchronous `readFileSync`, blocking the event loop for every request (metric or not) under load. Match the `.sql` path's behavior, then beat its per-request cost: - `loadMetricRegistry` is now async (`fs.promises`) and stays a pure, stateless parse. Metric views are already heavier than a plain query on the warehouse side, so the SDK layer must not add a blocking read. - New `getMetricRegistry(dir)` wraps it with a module-level cache keyed by the queries DIR (not the plugin instance): the registry is a pure function of the config file, warehouse-independent, so two plugins at one dir share one parse. Each request does a single async `stat`; the read + JSON.parse + zod validation are skipped when the file's (mtimeMs, size) signature is unchanged — steady-state cost is below the `.sql` path (a stat, not a full read). - Failures are NOT cached (cache populated only on a successful parse), so a fixed config self-heals on the next request; an edit bumps mtime so a working config hot-reloads; an absent file stays dormant (ENOENT → empty registry). - Deletes the `metricRegistry` / `metricRegistryLoadError` fields and the `_getMetricRegistry` memo; the route calls `getMetricRegistry` in a try/catch → 503 on throw. Registry loading is now exercised through real files: `AnalyticsPlugin` takes a test-only `queriesDir` config (documented `@internal`), so tests point the loader at a temp dir and drive the real stat→read→cache path instead of poking private state. Removes the `setRegistry` backdoor. Adds hot-reload, self-heal, and mtime-cache tests. Co-authored-by: Isaac Signed-off-by: Atila Fassina --- .../appkit/src/plugins/analytics/analytics.ts | 79 ++-- .../appkit/src/plugins/analytics/metric.ts | 126 +++++- .../plugins/analytics/tests/metric.test.ts | 406 +++++++++++++----- .../appkit/src/plugins/analytics/types.ts | 11 + 4 files changed, 454 insertions(+), 168 deletions(-) diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index c8bf5d4e..451373cc 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -32,8 +32,9 @@ import manifest from "./manifest.json"; import { buildMetricSql, composeMetricCacheKey, + QUERIES_DIR as DEFAULT_QUERIES_DIR, deriveMetricExecutorKey, - loadMetricRegistry, + getMetricRegistry, validateMetricRequest, } from "./metric"; import { QueryProcessor } from "./query"; @@ -125,28 +126,18 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { private _arrowCapability = new Map(); /** - * Metric-view registry parsed from `config/queries/metric-views.json`, keyed - * by metric key. Loaded lazily on the first `/metric/:key` request and - * memoized (see {@link _getMetricRegistry}). Empty when no config is present - * — the metric-view path stays dormant until an app opts in. + * Directory the metric-view registry is read from (holds + * `metric-views.json`). Defaults to `config/queries/` under the process cwd + * — the same directory the `.sql` query path uses. Overridable ONLY via + * `config.queriesDir` for tests (see {@link IAnalyticsConfig.queriesDir}). */ - private metricRegistry: Record | null = null; - - /** - * Latched error from the most recent {@link loadMetricRegistry} attempt. - * `null` means the registry loaded cleanly (or `metric-views.json` was absent - * — also fine; metric views are opt-in). When non-null, every `/metric/:key` - * request returns 503 `METRIC_REGISTRY_LOAD_FAILED` so a broken config - * (unreadable file, invalid JSON, schema violation) surfaces as a clear - * server status rather than masquerading as a 404 for every key. The full - * reason goes to telemetry only. - */ - private metricRegistryLoadError: string | null = null; + private readonly _queriesDir: string; constructor(config: IAnalyticsConfig) { super(config); this.config = config; this.queryProcessor = new QueryProcessor(); + this._queriesDir = config.queriesDir ?? DEFAULT_QUERIES_DIR; this.SQLClient = new SQLWarehouseConnector({ timeout: config.timeout, @@ -466,32 +457,6 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ); } - /** - * Lazily load and memoize the metric registry from - * `config/queries/metric-views.json`. - * - * A malformed config latches `metricRegistryLoadError` and yields an empty - * registry so the route can return a 503 (distinguishing a broken deployment - * from an unknown key, which is a 404). Loading is deferred to the first - * `/metric/:key` request rather than the constructor so apps that never adopt - * metric views pay no parse cost and a config error can't break plugin - * construction. - */ - private _getMetricRegistry(): Record { - if (this.metricRegistry === null) { - try { - this.metricRegistry = loadMetricRegistry(); - this.metricRegistryLoadError = null; - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); - logger.warn("Failed to load metric registry: %s", reason); - this.metricRegistry = {}; - this.metricRegistryLoadError = reason; - } - } - return this.metricRegistry; - } - /** * Handle metric-view execution requests (`POST /api/analytics/metric/:key`). * @@ -527,15 +492,20 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { return; } - const registry = this._getMetricRegistry(); - - // Surface a registry-load failure on the route. Without this, a malformed - // metric-views.json would yield 404 for every key — identical to "key - // never registered" — and hide the deployment error. Full reason → - // telemetry only. - if (this.metricRegistryLoadError !== null) { + // Resolve the registry from disk (cached + mtime-revalidated by + // `getMetricRegistry`, so the steady-state cost is one `stat`). A malformed + // config throws → 503, distinguishing a broken deployment from an unknown + // key (404). The failure is NOT latched: `getMetricRegistry` only caches on + // a successful parse, so fixing `metric-views.json` heals on the next + // request. Full reason → telemetry only. + let registry: Record; + try { + registry = await getMetricRegistry(this._queriesDir); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn(req, "Failed to load metric registry: %s", reason); event?.setContext("analytics", { - metric_registry_load_error: this.metricRegistryLoadError, + metric_registry_load_error: reason, }); res.status(503).json({ error: "Metric registry not available", @@ -546,10 +516,9 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { // Own-property lookup: never resolve `key` to an inherited // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …). - // The loader already builds a null-prototype registry, but the load-error - // fallback and test-injected registries are plain objects, so gate the - // read here too — an inherited hit would otherwise bypass the 404 below - // and flow a non-registration value into execution. + // `getMetricRegistry` returns a null-prototype map, but gate the read here + // too as defense-in-depth — an inherited hit would otherwise bypass the 404 + // below and flow a non-registration value into execution. const registration = Object.hasOwn(registry, key) ? registry[key] : undefined; diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index 5ed31946..a74c3cb4 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import fs from "node:fs"; +import fs from "node:fs/promises"; import path from "node:path"; import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; import { z } from "zod"; @@ -29,9 +29,10 @@ const logger = createLogger("analytics:metric"); /** * Default queries directory. Mirrors `AppManager`'s * `path.resolve(process.cwd(), "config/queries")` so dev mode and production - * share a single source of truth for where metric config lives. + * share a single source of truth for where metric config lives. Exported so + * `AnalyticsPlugin` can default `config.queriesDir` to the same path. */ -const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); const METRIC_CONFIG_FILE = "metric-views.json"; /** @@ -154,28 +155,33 @@ function laneFromExecutor( /** * Read and validate `config/queries/metric-views.json` into a metric registry. * - * Synchronous by design — registration is a pure config parse with no - * warehouse round-trip, no `DESCRIBE`, and no build-time metadata bundle. The - * single `metricViews` map makes keys unique by construction, so there is no - * cross-lane duplicate-key check. + * Async and stateless — registration is a pure config parse with no warehouse + * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single + * `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps + * the event loop free: metric views are already heavier than a plain `.sql` + * query on the warehouse side, so the SDK layer must not add a blocking read + * on top. Caching + mtime-revalidation is layered on top by + * {@link getMetricRegistry} — this function always hits disk. * * Returns an empty registry when the file is absent: the metric-view path is * additive and dormant until an app opts in by adding the config. A malformed * file (unreadable, invalid JSON, or schema violation) throws — the caller - * latches the failure so the route can surface a 503 rather than masking a - * broken deployment as a 404 for every key. + * surfaces a 503 rather than masking a broken deployment as a 404 for every + * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the + * file heals on the next request. */ -export function loadMetricRegistry( +export async function loadMetricRegistry( queriesDir: string = QUERIES_DIR, -): Record { +): Promise> { const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); let raw: string; try { - raw = fs.readFileSync(metricPath, "utf8"); + raw = await fs.readFile(metricPath, "utf8"); } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") { - return {}; + return Object.create(null); } throw err; } @@ -219,6 +225,100 @@ export function loadMetricRegistry( return registry; } +/** + * Signature of the config file the cached registry was parsed from: its + * modification time and size. Cheap to obtain (one `stat`) and sufficient to + * detect an edit — the same pair file-watching tools compare. A content hash + * would mean reading the file, which is exactly the cost the cache avoids. + */ +interface RegistryCacheSignature { + mtimeMs: number; + size: number; +} + +/** + * Module-level registry cache, keyed by the resolved queries directory. + * + * Keyed by DIR (not by plugin instance) because the registry is a pure + * function of the config file at that path — warehouse-independent, and two + * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see + * the same registry. Instance state would parse the identical file twice and + * risk divergence; a dir-keyed module cache shares one parse. + */ +const metricRegistryCache = new Map< + string, + { + signature: RegistryCacheSignature; + registry: Record; + } +>(); + +/** + * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only + * when `metric-views.json` has changed since the cached copy. + * + * Behaves like the sibling `.sql` query path (which re-reads per request) but + * cheaper: the steady-state cost is a single async `stat`, and the read + + * `JSON.parse` + zod validation are skipped when the file's `(mtimeMs, size)` + * signature is unchanged. This delivers the agreed semantics without the old + * permanent memo: + * + * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next + * request re-parses and serves the new registry with no server restart. + * - **Self-heal** — a load failure is NOT cached (we only populate the cache + * on a successful parse), so a fixed config is picked up on the next + * request instead of latching a 503 forever. + * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding + * the file later is picked up on the next request. + * + * Concurrency: two simultaneous cold requests may both `stat`+read before + * either populates the cache. That is a harmless redundant read of the same + * file (the sibling `.sql` path does not single-flight either), so no lock is + * taken. + * + * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so + * the route can surface a 503; the cache is left untouched on failure. + */ +export async function getMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let signature: RegistryCacheSignature | null; + try { + const stats = await fs.stat(metricPath); + signature = { mtimeMs: stats.mtimeMs, size: stats.size }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + // Absent file → dormant. Drop any stale cache entry (the file may have + // been deleted) and return an empty registry without touching the cache. + metricRegistryCache.delete(queriesDir); + signature = null; + } else { + throw err; + } + } + + if (signature === null) { + return Object.create(null); + } + + const cached = metricRegistryCache.get(queriesDir); + if ( + cached !== undefined && + cached.signature.mtimeMs === signature.mtimeMs && + cached.signature.size === signature.size + ) { + return cached.registry; + } + + // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed + // file never latches — the next request re-attempts and heals. + const registry = await loadMetricRegistry(queriesDir); + metricRegistryCache.set(queriesDir, { signature, registry }); + return registry; +} + /** * Validate a metric-view FQN and return it backtick-quoted for interpolation. * diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index cde55285..598bf932 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -16,6 +16,7 @@ import { buildMetricSql, composeMetricCacheKey, deriveMetricExecutorKey, + getMetricRegistry, loadMetricRegistry, validateMetricRequest, } from "../metric"; @@ -60,13 +61,58 @@ vi.mock("../../../cache", () => ({ }, })); -/** Feed the plugin a registry directly, bypassing the disk config parse. */ -function setRegistry( - plugin: AnalyticsPlugin, - registry: Record, +// Temp dirs created by `registryDir` / `writeRegistry`, cleaned up after each +// test. Using real files (via the test-only `queriesDir` config) exercises the +// actual stat → read → cache path in `getMetricRegistry` rather than poking +// private plugin state. +const tempRegistryDirs: string[] = []; + +/** + * Write a `metric-views.json` into a fresh temp dir and return the dir, for use + * as `new AnalyticsPlugin({ ...config, queriesDir })`. Accepts the internal + * `MetricRegistration` shape (matching the old `setRegistry` helper) and maps + * each entry's `lane` back to the config's `executor` field. + */ +function registryDir(registry: Record): string { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + const metricViews: Record = {}; + for (const [key, reg] of Object.entries(registry)) { + metricViews[key] = { + source: reg.source, + executor: reg.lane === "obo" ? "user" : "app_service_principal", + }; + } + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + return dir; +} + +/** + * Overwrite the `metric-views.json` in an existing temp dir (for hot-reload / + * self-heal tests). `raw` lets a test write deliberately malformed content. + */ +function writeRegistry( + dir: string, + content: Record | string, ): void { - (plugin as any).metricRegistry = registry; - (plugin as any).metricRegistryLoadError = null; + const body = + typeof content === "string" + ? content + : JSON.stringify({ + metricViews: Object.fromEntries( + Object.entries(content).map(([key, reg]) => [ + key, + { + source: reg.source, + executor: reg.lane === "obo" ? "user" : "app_service_principal", + }, + ]), + ), + }); + writeFileSync(path.join(dir, "metric-views.json"), body); } describe("analytics metric route (Phase 1)", () => { @@ -83,6 +129,10 @@ describe("analytics metric route (Phase 1)", () => { afterEach(() => { serviceContextMock?.restore(); + while (tempRegistryDirs.length > 0) { + const dir = tempRegistryDirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } }); describe("injectRoutes", () => { @@ -365,15 +415,17 @@ describe("analytics metric route (Phase 1)", () => { // byte-identical to the /query route's JSON SSE path. describe("_handleMetricRoute SSE envelope", () => { test("streams warehouse_status then a result message with aliased rows", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn().mockResolvedValue({ result: { data: [{ arr: 1234 }] }, @@ -410,15 +462,17 @@ describe("analytics metric route (Phase 1)", () => { }); test("emits warehouse_status before result for a STARTING warehouse", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn().mockResolvedValue({ result: { data: [{ arr: 1 }] }, @@ -461,15 +515,17 @@ describe("analytics metric route (Phase 1)", () => { }); test("returns 400 when the body fails structural validation", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); const handler = getHandler("POST", "/metric/:key"); @@ -488,15 +544,17 @@ describe("analytics metric route (Phase 1)", () => { // ── 503-vs-404 latching + dormancy. describe("registry latching and dormancy", () => { test("unknown key against a valid registry → 404 (generic body)", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); plugin.injectRoutes(router); const handler = getHandler("POST", "/metric/:key"); @@ -515,18 +573,20 @@ describe("analytics metric route (Phase 1)", () => { test.each(["__proto__", "constructor", "toString", "hasOwnProperty"])( "inherited Object.prototype key %j → 404, no execution (own-property lookup)", async (dangerousKey) => { - const plugin = new AnalyticsPlugin(config); + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); const { router, getHandler } = createMockRouter(); // A real (own) entry so the registry is populated but does NOT contain // the dangerous key. Without the own-property guard, `registry[key]` // would resolve the inherited prototype member and slip past the 404. - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, - }); const executeMock = vi.fn(); (plugin as any).SQLClient.executeStatement = executeMock; @@ -550,11 +610,13 @@ describe("analytics metric route (Phase 1)", () => { ); test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { - const plugin = new AnalyticsPlugin(config); + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + // A real malformed config on disk (invalid JSON) → the loader throws → + // the route surfaces a 503, exercising the actual load path. + writeRegistry(dir, "{ not valid json"); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); const { router, getHandler } = createMockRouter(); - // Latch a load error directly (as a malformed config would). - (plugin as any).metricRegistry = {}; - (plugin as any).metricRegistryLoadError = "Invalid metric-views.json"; plugin.injectRoutes(router); const handler = getHandler("POST", "/metric/:key"); @@ -573,6 +635,104 @@ describe("analytics metric route (Phase 1)", () => { }); }); + test("self-heal: a fixed config is picked up on the next request (no restart)", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + writeRegistry(dir, "{ not valid json"); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + // First request: malformed → 503, and the failure is NOT cached. + const res1 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }), + res1, + ); + expect(res1.status).toHaveBeenCalledWith(503); + + // Fix the file. mtime changes, so the next request re-parses and serves + // it — no server restart, no latched 503. + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + const res2 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }), + res2, + ); + expect(res2.status).not.toHaveBeenCalledWith(503); + expect(res2.status).not.toHaveBeenCalledWith(404); + expect(executeMock).toHaveBeenCalled(); + }); + + test("hot-reload: a new key added to a working config is visible next request", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + // `orders` is not registered yet → 404. + const res1 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "orders" }, + body: { measures: ["cnt"] }, + }), + res1, + ); + expect(res1.status).toHaveBeenCalledWith(404); + + // Add `orders` to the config. The mtime bump invalidates the cache, so + // the next request sees it without a restart. + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + orders: { key: "orders", source: "cat.sch.order_metrics", lane: "sp" }, + }); + const res2 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "orders" }, + body: { measures: ["cnt"] }, + }), + res2, + ); + expect(res2.status).not.toHaveBeenCalledWith(404); + expect(executeMock).toHaveBeenCalled(); + }); + test("no metric-views.json present → registry empty, unknown key 404, nothing executes", async () => { const plugin = new AnalyticsPlugin(config); const { router, getHandler } = createMockRouter(); @@ -609,11 +769,11 @@ describe("loadMetricRegistry", () => { rmSync(dir, { recursive: true, force: true }); }); - test("absent metric-views.json → empty registry (dormancy)", () => { - expect(loadMetricRegistry(dir)).toEqual({}); + test("absent metric-views.json → empty registry (dormancy)", async () => { + expect(await loadMetricRegistry(dir)).toEqual({}); }); - test("derives lane from executor (default sp, user → obo)", () => { + test("derives lane from executor (default sp, user → obo)", async () => { writeFileSync( path.join(dir, "metric-views.json"), JSON.stringify({ @@ -624,7 +784,7 @@ describe("loadMetricRegistry", () => { }), ); - const registry = loadMetricRegistry(dir); + const registry = await loadMetricRegistry(dir); expect(registry.revenue).toEqual({ key: "revenue", source: "cat.sch.revenue_metrics", @@ -637,7 +797,7 @@ describe("loadMetricRegistry", () => { }); }); - test("registry has a null prototype (no inherited-property lookups)", () => { + test("registry has a null prototype (no inherited-property lookups)", async () => { writeFileSync( path.join(dir, "metric-views.json"), JSON.stringify({ @@ -645,7 +805,7 @@ describe("loadMetricRegistry", () => { }), ); - const registry = loadMetricRegistry(dir); + const registry = await loadMetricRegistry(dir); expect(Object.getPrototypeOf(registry)).toBeNull(); // A key that would resolve to a truthy inherited member on a plain object // resolves to undefined here. @@ -653,19 +813,55 @@ describe("loadMetricRegistry", () => { expect((registry as Record).__proto__).toBeUndefined(); }); - test("malformed JSON throws", () => { + test("absent file yields a null-prototype (dormant) registry too", async () => { + // The ENOENT dormancy path must also be prototype-free — a metric key + // colliding with an inherited member can't 200 against an empty registry. + const registry = await loadMetricRegistry(dir); + expect(Object.getPrototypeOf(registry)).toBeNull(); + }); + + test("malformed JSON throws", async () => { writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); - expect(() => loadMetricRegistry(dir)).toThrow(/Failed to parse/); + await expect(loadMetricRegistry(dir)).rejects.toThrow(/Failed to parse/); }); - test("schema-invalid config throws", () => { + test("schema-invalid config throws", async () => { writeFileSync( path.join(dir, "metric-views.json"), JSON.stringify({ metricViews: { revenue: { source: "not-a-three-part-fqn" } }, }), ); - expect(() => loadMetricRegistry(dir)).toThrow(/Invalid metric-views.json/); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("re-reads only after the file changes (mtime-validated cache)", async () => { + // getMetricRegistry caches by dir and revalidates via stat; a second call + // with no change returns the same object, and a rewrite is picked up. + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + const first = await getMetricRegistry(dir); + const second = await getMetricRegistry(dir); + expect(second).toBe(first); // same cached object, no re-parse + + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { + revenue: { source: "cat.sch.revenue_metrics" }, + orders: { source: "cat.sch.order_metrics" }, + }, + }), + ); + const third = await getMetricRegistry(dir); + expect(third).not.toBe(first); + expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); }); }); @@ -1375,15 +1571,17 @@ describe("metric — filter translator", () => { }); test("a well-formed-but-unknown measure reaches the warehouse and surfaces a sanitized error envelope", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); // The warehouse rejects the unknown column. The raw text carries the // offending name; the connector wraps it in an ExecutionError whose @@ -1755,15 +1953,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO-lane registration routes through asUser(req)", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "obo", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), }); + const { router, getHandler } = createMockRouter(); const asUserSpy = vi.spyOn(plugin, "asUser"); const executeMock = vi @@ -1793,15 +1993,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("SP-lane registration uses the default executor (asUser not called)", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "sp", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), }); + const { router, getHandler } = createMockRouter(); const asUserSpy = vi.spyOn(plugin, "asUser"); const executeMock = vi @@ -1830,15 +2032,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO metric with no user identity → canonical 401, no SQL executed", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "obo", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn(); (plugin as any).SQLClient.executeStatement = executeMock; @@ -1865,15 +2069,17 @@ describe("metric route — lane dispatch (Phase 3)", () => { }); test("OBO metric with whitespace-only user identity → canonical 401", async () => { - const plugin = new AnalyticsPlugin(config); - const { router, getHandler } = createMockRouter(); - setRegistry(plugin, { - revenue: { - key: "revenue", - source: "cat.sch.revenue_metrics", - lane: "obo", - }, + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), }); + const { router, getHandler } = createMockRouter(); const executeMock = vi.fn(); (plugin as any).SQLClient.executeStatement = executeMock; diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index e91a361f..c01c04e2 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -23,6 +23,17 @@ export interface IAnalyticsConfig extends BasePluginConfig { * byte arrives the stream is not time-bounded. */ arrowFirstByteTimeoutMs?: number; + /** + * Directory to read the metric-view registry (`metric-views.json`) from. + * + * @internal FOR TESTS ONLY. Production always reads the fixed + * `config/queries/` path (the default), the same directory the `.sql` query + * path uses — do NOT set this to relocate config in a real app. It exists so + * tests can point the loader at a temp directory and exercise the real + * stat → read → cache path against a real file, rather than poking private + * plugin state or stubbing the loader. + */ + queriesDir?: string; } /** From f53e1702ade3bdb4d0906553d02559f2e3614a4b Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 12:36:11 +0200 Subject: [PATCH 8/9] fix(appkit): quote measure/dimension identifiers + close review-round-4 gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third adversarial review round (two reviewers, deduped). Highest-priority finding: the FQN grammar drift fixed earlier was still present for measures/dimensions/filter-members — the type-generator emits DESCRIBE column names verbatim into the generated measureKeys/dimensionKeys unions, but the runtime gated them on the narrow /^[a-zA-Z_][a-zA-Z0-9_]*$/ and interpolated bare, so a UC column like `net-revenue` / `café_sales` typechecked yet 500'd at runtime (generate-but-500, same class as the FQN bug). A+C — quote column identifiers + validate early: - Add `isValidColumnName` + `quoteIdentifier` to the shared zod-free leaf. `quoteIdentifier` is the single-identifier escaper (does NOT split on `.`, so a column literally named `net.revenue` becomes one delimited identifier); `quoteFqnForSql` now maps it over dot-split segments. The column grammar is the full delimited-identifier set — reject only control/newline (what cannot be safely quoted), matching exactly what typegen can emit. - Runtime backtick-quotes measures (MEASURE(`x`) AS `x`), dimensions, the date_trunc column, and filter members. Quoting — not a narrow allowlist — is the injection boundary, so an injection-shaped name is neutralized (inert quoted column), not rejected. Row-key preservation holds: the warehouse reports the aliased column under the unquoted name, so `{ "net-revenue": … }`. - Move identifier validation into `validateMetricRequest` (via a refine on measures/dimensions/timeDimension/filter.member) so a malformed identifier returns the canonical 400 instead of failing inside the SSE execute path as a retried 500 (finding C). The builder keeps its checks as defense-in-depth. - Delete the now-unused MEASURE_NAME_PATTERN / DIMENSION_NAME_PATTERN. - Fixed a regression this introduces: the filter sort key and canonicalize fingerprint used a "/" delimiter that was safe only while members couldn't contain "/"; now that members accept the full grammar, both JSON-encode the tuple so distinct (member, operator[, values]) pairs can't collide and fork/merge cache entries. B — registry cache signature adds ctimeMs (from the same stat): a same-size, same-mtime edit (equal-length source/executor swap on a coarse-mtime FS) now invalidates, closing a stale-source/stale-lane serve. + same-size regression test. D — runtime config caps now match the type-generator: MAX_METRIC_VIEWS (200) and per-segment length (255) enforced via the shared schema's superRefine, plus a declarative maxLength (767) on `source` (the only cap `z.toJSONSchema` can serialize — regenerated metric-source.schema.json). Refinements are invisible to JSON-schema generation, so runtime + typegen stay the authoritative gates. E — export test-only `__resetMetricRegistryCache` so cache isolation between tests is intentional, not an accident of unique temp dirs. F — comment that a non-ENOENT stat error is deliberately fatal-per-request (self-heals next call). Co-authored-by: Isaac Signed-off-by: Atila Fassina --- docs/static/schemas/metric-source.schema.json | 1 + .../appkit/src/plugins/analytics/metric.ts | 221 +++++++++------ .../plugins/analytics/tests/metric.test.ts | 264 +++++++++++++----- packages/shared/src/schemas/metric-fqn.ts | 67 +++-- packages/shared/src/schemas/metric-source.ts | 55 +++- 5 files changed, 446 insertions(+), 162 deletions(-) diff --git a/docs/static/schemas/metric-source.schema.json b/docs/static/schemas/metric-source.schema.json index 1976ea7c..e606b92d 100644 --- a/docs/static/schemas/metric-source.schema.json +++ b/docs/static/schemas/metric-source.schema.json @@ -21,6 +21,7 @@ "properties": { "source": { "type": "string", + "maxLength": 767, "pattern": "^[^\\x00-\\x20\\x7f./]+\\.[^\\x00-\\x20\\x7f./]+\\.[^\\x00-\\x20\\x7f./]+$", "description": "Three-part Unity Catalog FQN of the metric view: ..", "examples": [ diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index a74c3cb4..aabfdc89 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -8,8 +8,10 @@ import { z } from "zod"; // type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the // same tree) so the runtime and the generated JSON schema validate identically. import { + isValidColumnName, isValidFqn, quoteFqnForSql, + quoteIdentifier, } from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { AuthenticationError, ValidationError } from "../../errors"; @@ -36,32 +38,20 @@ export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); const METRIC_CONFIG_FILE = "metric-views.json"; /** - * Validate measure names before they are interpolated into `MEASURE()`. + * Measure, dimension, and filter-member names are **column identifiers**: they + * are validated by the shared {@link isValidColumnName} (rejects only control + * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at + * every interpolation point. Quoting — not a narrow ASCII allowlist — is the + * injection boundary, so the runtime accepts the full delimited-identifier + * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). + * There is deliberately NO name allowlist: a well-formed-but-unknown column + * falls through to the warehouse and surfaces as a sanitized canonical error. * - * Measure names cannot be parameterized — they are SQL identifiers, not - * literals. This conservative identifier shape is the security boundary for - * the interpolated tokens: there is deliberately NO name allowlist, so a - * well-formed-but-unknown measure falls through to the warehouse and surfaces - * as a sanitized canonical error (parity with the raw `.sql` flow). - */ -const MEASURE_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - -/** - * Dimension name pattern. Matches the identifier shape we accept for measures - * — column references cannot be parameterized in SQL, so they must be - * conservatively safe identifiers (no spaces, no quotes, no SQL operators). - * This grammar gate is the security boundary for interpolated dimension - * tokens: there is deliberately NO name allowlist, so a well-formed-but-unknown - * dimension falls through to the warehouse and surfaces as a sanitized - * canonical error. - */ -const DIMENSION_NAME_PATTERN = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - -/** - * Time-grain token shape. This grammar gate is the security boundary for the - * grain: it is interpolated as a single-quoted `date_trunc` unit literal (NOT - * a bind param) in {@link renderDimensionClause}, so the pattern is what keeps - * a hostile token out of that quoted position. + * Time-grain token shape. Unlike the column identifiers above, the grain is + * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, + * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a + * narrow keyword-shaped gate — that pattern is what keeps a hostile token out + * of the quoted-literal position. */ const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; @@ -227,11 +217,22 @@ export async function loadMetricRegistry( /** * Signature of the config file the cached registry was parsed from: its - * modification time and size. Cheap to obtain (one `stat`) and sufficient to - * detect an edit — the same pair file-watching tools compare. A content hash - * would mean reading the file, which is exactly the cost the cache avoids. + * change time, modification time, and size — all from one `stat`, no read. + * + * `ctimeMs` (inode change time) is the key guard against a stale serve: it + * bumps on ANY metadata or content change and, unlike `mtimeMs`, cannot be + * restored to a prior value by tooling (`utimes` sets atime/mtime but not + * ctime). So an edit that preserves both size and mtime — a same-length value + * swap (e.g. repointing `source` to an equal-length FQN, or flipping + * `executor` between two equal-length values) on a coarse-mtime filesystem — + * still changes ctime and invalidates the cache. Without it, such an edit + * could serve a stale `source`/`lane`, reintroducing the exact stale-serve + * class the cache-key `source` salt was added to prevent, one layer up. + * A content hash would be stronger still but requires reading the file, which + * is the cost this cache exists to avoid. */ interface RegistryCacheSignature { + ctimeMs: number; mtimeMs: number; size: number; } @@ -253,15 +254,28 @@ const metricRegistryCache = new Map< } >(); +/** + * Clear the module-level registry cache. + * + * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the + * process; production never needs to clear it (a changed file is picked up via + * the stat signature). Tests that reuse a directory — or assert cold-load + * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional + * rather than relying on each test happening to use a unique temp dir. + */ +export function __resetMetricRegistryCache(): void { + metricRegistryCache.clear(); +} + /** * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only * when `metric-views.json` has changed since the cached copy. * * Behaves like the sibling `.sql` query path (which re-reads per request) but * cheaper: the steady-state cost is a single async `stat`, and the read + - * `JSON.parse` + zod validation are skipped when the file's `(mtimeMs, size)` - * signature is unchanged. This delivers the agreed semantics without the old - * permanent memo: + * `JSON.parse` + zod validation are skipped when the file's + * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed + * semantics without the old permanent memo: * * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next * request re-parses and serves the new registry with no server restart. @@ -287,7 +301,11 @@ export async function getMetricRegistry( let signature: RegistryCacheSignature | null; try { const stats = await fs.stat(metricPath); - signature = { mtimeMs: stats.mtimeMs, size: stats.size }; + signature = { + ctimeMs: stats.ctimeMs, + mtimeMs: stats.mtimeMs, + size: stats.size, + }; } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") { // Absent file → dormant. Drop any stale cache entry (the file may have @@ -295,6 +313,11 @@ export async function getMetricRegistry( metricRegistryCache.delete(queriesDir); signature = null; } else { + // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal + // for THIS request → the route surfaces a 503, consistent with the + // malformed-config → 503 path. It is not latched: with the self-heal + // design a transient error clears on the next request, which is strictly + // better than the old memo that could latch-and-serve-stale. throw err; } } @@ -306,6 +329,7 @@ export async function getMetricRegistry( const cached = metricRegistryCache.get(queriesDir); if ( cached !== undefined && + cached.signature.ctimeMs === signature.ctimeMs && cached.signature.mtimeMs === signature.mtimeMs && cached.signature.size === signature.size ) { @@ -374,7 +398,11 @@ const filterPredicateSchema: z.ZodType = z .object({ member: z .string() - .min(1, { message: "filter predicate 'member' cannot be empty" }), + .min(1, { message: "filter predicate 'member' cannot be empty" }) + .refine(isValidColumnName, { + message: + "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", + }), operator: z.string().min(1, { message: "filter predicate 'operator' cannot be empty", }) as z.ZodType, @@ -411,13 +439,29 @@ const filterSchema: z.ZodType = z.lazy(() => const metricRequestSchema = z .object({ measures: z - .array(z.string().min(1, "measure name cannot be empty")) + .array( + z + .string() + .min(1, "measure name cannot be empty") + .refine(isValidColumnName, { + message: + "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) .min(1, "at least one measure is required") .max(METRIC_MEASURES_MAX, { message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, }), dimensions: z - .array(z.string().min(1, "dimension name cannot be empty")) + .array( + z + .string() + .min(1, "dimension name cannot be empty") + .refine(isValidColumnName, { + message: + "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) .max(METRIC_DIMENSIONS_MAX, { message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, }) @@ -434,15 +478,16 @@ const metricRequestSchema = z message: "timeGrain must match /^[a-z][a-z_]*$/", }) .optional(), - // The single dimension `timeGrain` applies to via `date_trunc`. Grammar- - // gated as a SQL identifier (column reference, cannot be parameterized). - // Cross-field rules in `superRefine`: required when `timeGrain` is set, and - // must be one of `dimensions`. + // The single dimension `timeGrain` applies to via `date_trunc`. A column + // identifier (backtick-quoted at interpolation), so it accepts the full + // delimited-identifier grammar. Cross-field rules in `superRefine`: + // required when `timeGrain` is set, and must be one of `dimensions`. timeDimension: z .string() .min(1, { message: "timeDimension cannot be empty" }) - .regex(DIMENSION_NAME_PATTERN, { - message: "timeDimension must match /^[a-zA-Z_][a-zA-Z0-9_]*$/", + .refine(isValidColumnName, { + message: + "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", }) .optional(), limit: z @@ -765,12 +810,12 @@ export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { * [LIMIT n] * * Notes: - * - Every measure and dimension is gated by {@link MEASURE_NAME_PATTERN} / - * {@link DIMENSION_NAME_PATTERN} before it is interpolated (column - * references cannot be parameterized — they are SQL identifiers), and the - * FQN is validated and backtick-quoted by {@link quoteSafeFqn}. There is - * deliberately NO name allowlist — the grammar gate (plus quoting for the - * FQN) is the security boundary. No user-supplied string reaches the SQL + * - Every measure and dimension is validated by {@link isValidColumnName} and + * backtick-quoted by {@link quoteIdentifier} before it is interpolated + * (column references cannot be parameterized — they are SQL identifiers), + * and the FQN is validated and quoted by {@link quoteSafeFqn}. There is + * deliberately NO name allowlist — quoting is the security boundary. No + * user-supplied string reaches the SQL * string without passing a grammar gate. * - `GROUP BY ALL` is added when at least one dimension is requested. UC * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; @@ -800,8 +845,13 @@ export function buildMetricSql( throw new Error("buildMetricSql requires at least one measure."); } + // Defense-in-depth re-gate: `validateMetricRequest` already rejects any name + // `isValidColumnName` refuses, but `buildMetricSql` is exported and may be + // reached without it. `quoteIdentifier` throws on a control/newline name, so + // the quoting below is itself the boundary; the explicit check keeps the + // error message uniform with the other interpolated tokens. for (const m of request.measures) { - if (!MEASURE_NAME_PATTERN.test(m)) { + if (!isValidColumnName(m)) { throw new Error( `Refusing to build SQL: measure "${m}" is not a valid identifier.`, ); @@ -810,7 +860,7 @@ export function buildMetricSql( const dimensions = request.dimensions ?? []; for (const d of dimensions) { - if (!DIMENSION_NAME_PATTERN.test(d)) { + if (!isValidColumnName(d)) { throw new Error( `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, ); @@ -818,12 +868,15 @@ export function buildMetricSql( } // Deterministic order so cache keys collapse semantically equivalent calls. - // Alias each measure to its plain name so result rows have keys matching the - // registered measure (`{ arr: 1234 }`) rather than the SQL-function - // serialization Databricks returns by default (`{ "measure(arr)": 1234 }`). + // Alias each measure to its plain name (backtick-quoted) so result rows have + // keys matching the registered measure (`{ "net-revenue": 1234 }`) rather + // than the SQL-function serialization Databricks returns by default. The + // warehouse reports the aliased column under the UNQUOTED name (backticks are + // delimiters, not part of the name), so `MEASURE(`x`) AS `x`` yields a row + // key of exactly `x` — preserved through result materialization. const measureClauses = [...request.measures] .sort() - .map((m) => `MEASURE(${m}) AS ${m}`); + .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); const dimensionClauses = [...dimensions] .sort() @@ -880,9 +933,9 @@ interface FilterRenderState { * validator bypass cannot turn `or: []` into "match everything"). * * Defense-in-depth: even though the request body's filter has already been - * validated, every member name is re-checked against {@link - * DIMENSION_NAME_PATTERN} here. If validation is ever bypassed, the SQL - * constructor still refuses to interpolate an unknown-shaped identifier. + * validated, every member name is re-checked against {@link isValidColumnName} + * here and backtick-quoted. If validation is ever bypassed, the SQL + * constructor still refuses to interpolate a name it cannot safely quote. */ function renderFilter( node: MetricFilter, @@ -944,7 +997,7 @@ function renderFilter( // Leaf predicate — grammar-gate the member and operator, then render. const predicate = node as MetricPredicate; - if (!DIMENSION_NAME_PATTERN.test(predicate.member)) { + if (!isValidColumnName(predicate.member)) { throw new Error( `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, ); @@ -984,15 +1037,15 @@ function sortFilterChildren( !("or" in child) ) { const p = child as MetricPredicate; - // Separate the fields with "/" so the (member, operator) pair maps to - // the sort key injectively. A delimiter-free concatenation collides — - // member "ab" + op "c" and member "a" + op "bc" both → "abc" — which - // tie-breaks two distinct pairs to input order and forks the cache key - // on semantically equal filters. "/" can never appear in either field - // (members match DIMENSION_NAME_PATTERN, operators are the alpha-only - // enum), matching the leaf-fingerprint delimiter canonicalizeFilter - // already uses. - key = `${p.member}/${p.operator}`; + // JSON.stringify the (member, operator) pair so the sort key is + // injective regardless of content. A plain delimiter is unsafe now that + // members accept the full delimited-identifier grammar (a member may + // contain "/", ".", etc.): `member "a/b" + op "c"` and + // `member "a" + op "b/c"` would both collapse to "a/b/c" under any + // single-char separator, tie-breaking distinct pairs to input order. + // JSON encoding escapes any separator, so distinct pairs map to distinct + // keys — the same technique canonicalizeFilter uses for values. + key = JSON.stringify([p.member, p.operator]); isPredicate = true; } else { // Nested groups don't have a single (member, operator) — keep their @@ -1034,7 +1087,11 @@ function renderPredicate( params: Record, state: FilterRenderState, ): string { - const col = predicate.member; + // Backtick-quote the column identifier (defense-in-depth: the member was + // already validated by `isValidColumnName` in `renderFilter`). Quoting is + // what lets a delimited column name — hyphens, dots, non-ASCII — reach SQL + // safely; the bound value still flows through `:f_` params, never here. + const col = quoteIdentifier(predicate.member); const op = predicate.operator; const values = predicate.values ?? []; @@ -1153,8 +1210,8 @@ function bindLikeValue( * every other interpolated identifier (measures, dimensions, `timeDimension`, * the FQN) is re-gated in the builder. * - The column cannot be parameterized (it is an identifier), so it is - * re-checked against {@link DIMENSION_NAME_PATTERN} here as belt-and- - * suspenders even though the schema already gated `timeDimension`. + * re-checked against {@link isValidColumnName} and backtick-quoted here as + * belt-and-suspenders even though the schema already gated `timeDimension`. * * The aliasing keeps result-row keys stable (`{ order_date: ... }`) regardless * of whether the grain was applied. @@ -1165,19 +1222,23 @@ function renderDimensionClause( timeDimension: string | undefined, ): string { if (timeGrain != null && dim === timeDimension) { - if (!DIMENSION_NAME_PATTERN.test(dim)) { + if (!isValidColumnName(dim)) { throw new Error( `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, ); } + // The grain is interpolated as a single-quoted `date_trunc` unit literal + // (NOT a bind param), so it stays gated by the narrow TIME_GRAIN_PATTERN — + // it is a keyword, not a delimited identifier. if (!TIME_GRAIN_PATTERN.test(timeGrain)) { throw new Error( `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, ); } - return `date_trunc('${timeGrain}', ${dim}) AS ${dim}`; + const quoted = quoteIdentifier(dim); + return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; } - return dim; + return quoteIdentifier(dim); } /** @@ -1318,12 +1379,14 @@ function canonicalizeFilter(node: MetricFilter): string { return `${groupKey}(${childFingerprints.join(",")})`; } - // Leaf predicate. Use JSON.stringify (not String) for the value segment so - // strings carrying the `|` separator cannot collide with split arrays — - // e.g. `["a", "b"]` and `["a|string:b"]` stay distinct fingerprints. + // Leaf predicate. JSON.stringify every structural part — member, operator, + // and each value (with its type) — so no content can be confused with a + // separator. This matters now that `member` accepts the full delimited- + // identifier grammar (it may contain "/", ".", etc.): a bare + // `p(${member}/${operator}/...)` would let `member "a", op "b"` collide with + // `member "a/b"` and fork/merge cache entries. Encoding the whole tuple as + // JSON makes the fingerprint injective regardless of member/value content. const p = node as MetricPredicate; - const valuesPart = p.values - ? p.values.map((v) => `${typeof v}:${JSON.stringify(v)}`).join("|") - : ""; - return `p(${p.member}/${p.operator}/${valuesPart})`; + const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); + return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; } diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts index 598bf932..45ed380f 100644 --- a/packages/appkit/src/plugins/analytics/tests/metric.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { @@ -13,6 +13,7 @@ import { ServiceContext } from "../../../context/service-context"; import { AuthenticationError } from "../../../errors"; import { AnalyticsPlugin } from "../analytics"; import { + __resetMetricRegistryCache, buildMetricSql, composeMetricCacheKey, deriveMetricExecutorKey, @@ -129,6 +130,7 @@ describe("analytics metric route (Phase 1)", () => { afterEach(() => { serviceContextMock?.restore(); + __resetMetricRegistryCache(); while (tempRegistryDirs.length > 0) { const dir = tempRegistryDirs.pop(); if (dir) rmSync(dir, { recursive: true, force: true }); @@ -149,28 +151,46 @@ describe("analytics metric route (Phase 1)", () => { }); }); - // ── Grammar gate — the primary security test (replaces #341's - // fail-closed-503 allowlist test). A measure that fails MEASURE_NAME_PATTERN - // throws inside buildMetricSql BEFORE any SQL string is constructed. - describe("buildMetricSql grammar gate", () => { + // ── Identifier safety — the primary security test. Measure/dimension names + // are backtick-quoted (not narrow-gated) at interpolation, so an injection- + // shaped name is NEUTRALIZED by quoting rather than rejected: it becomes an + // inert (if nonexistent) column the warehouse resolves away. Only a name + // that cannot be safely quoted at all — one containing a control character + // or newline — is refused. + describe("buildMetricSql identifier safety (quoting)", () => { const registration: MetricRegistration = { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }; - test("throws before building SQL for an injection-shaped measure", () => { - expect(() => - buildMetricSql(registration, { - measures: ["arr; DROP TABLE users"], - }), - ).toThrow(/not a valid identifier/); + test("neutralizes an injection-shaped measure by quoting (no breakout)", () => { + // `arr; DROP TABLE users` has no control chars, so it is a valid (if + // nonexistent) column name: quoted whole, the `;` and `DROP` are inert + // inside the backtick-delimited identifier — not separate statements. + const { statement } = buildMetricSql(registration, { + measures: ["arr; DROP TABLE users"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr; DROP TABLE users`) AS `arr; DROP TABLE users` FROM `cat`.`sch`.`revenue_metrics`", + ); }); - test("throws for a measure with a backtick / quote", () => { + test("neutralizes a backtick in a measure by doubling it", () => { + // The one real breakout char is the backtick; quoteIdentifier doubles it + // so it cannot close the identifier early. + const { statement } = buildMetricSql(registration, { + measures: ["arr`"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr```) AS `arr``` FROM `cat`.`sch`.`revenue_metrics`", + ); + }); + + test("throws for a measure containing a control character (cannot be quoted)", () => { expect(() => - buildMetricSql(registration, { measures: ["arr`"] }), - ).toThrow(/not a valid identifier/); + buildMetricSql(registration, { measures: ["arr\ndrop"] }), + ).toThrow(/not a valid identifier|control character/); }); test("throws when no measures are supplied", () => { @@ -200,7 +220,7 @@ describe("analytics metric route (Phase 1)", () => { { measures: ["arr"] }, ); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `prod-data`.`analytics`.`revenue`", + "SELECT MEASURE(`arr`) AS `arr` FROM `prod-data`.`analytics`.`revenue`", ); }); @@ -214,7 +234,7 @@ describe("analytics metric route (Phase 1)", () => { { measures: ["arr"] }, ); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`re``v`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`re``v`", ); }); }); @@ -227,12 +247,12 @@ describe("analytics metric route (Phase 1)", () => { lane: "sp", }; - test("single measure → SELECT MEASURE(m) AS m FROM ", () => { + test("single measure → SELECT MEASURE(`m`) AS `m` FROM ", () => { const { statement, parameters } = buildMetricSql(registration, { measures: ["arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", ); expect(parameters).toEqual({}); }); @@ -242,7 +262,7 @@ describe("analytics metric route (Phase 1)", () => { measures: ["revenue", "arr"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue` FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -252,7 +272,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 10.9, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", ); }); }); @@ -272,7 +292,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -282,7 +302,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["segment", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region, segment FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, `region`, `segment` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -292,7 +312,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, MEASURE(revenue) AS revenue, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -302,7 +322,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: [], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", ); }); @@ -313,7 +333,7 @@ describe("analytics metric route (Phase 1)", () => { limit: 100, }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", + "SELECT MEASURE(`arr`) AS `arr`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", ); }); @@ -323,7 +343,7 @@ describe("analytics metric route (Phase 1)", () => { dimensions: ["order_date", "region"], }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, `order_date`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).not.toContain("date_trunc"); }); @@ -347,10 +367,10 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('month', order_date) AS order_date, region FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, date_trunc('month', `order_date`) AS `order_date`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); expect(statement).toContain( - "date_trunc('month', order_date) AS order_date", + "date_trunc('month', `order_date`) AS `order_date`", ); expect(statement).toContain(" GROUP BY ALL"); }); @@ -363,7 +383,7 @@ describe("analytics metric route (Phase 1)", () => { timeDimension: "order_date", }); expect(statement).toBe( - "SELECT MEASURE(arr) AS arr, date_trunc('day', order_date) AS order_date FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + "SELECT MEASURE(`arr`) AS `arr`, date_trunc('day', `order_date`) AS `order_date` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", ); }); @@ -383,31 +403,43 @@ describe("analytics metric route (Phase 1)", () => { }); }); - // ── Phase 2: dimension grammar gate. A dimension failing - // DIMENSION_NAME_PATTERN throws inside buildMetricSql before SQL is built. - describe("buildMetricSql dimension grammar gate", () => { + // ── Phase 2: dimension identifier safety. A dimension is backtick-quoted at + // interpolation, so an injection-shaped name is neutralized (inert quoted + // column), and only an unquotable (control-char) name throws. + describe("buildMetricSql dimension identifier safety (quoting)", () => { const registration: MetricRegistration = { key: "revenue", source: "cat.sch.revenue_metrics", lane: "sp", }; - test("throws before building SQL for an injection-shaped dimension", () => { - expect(() => - buildMetricSql(registration, { - measures: ["arr"], - dimensions: ["region; DROP TABLE users"], - }), - ).toThrow(/not a valid identifier/); + test("neutralizes an injection-shaped dimension by quoting", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region; DROP TABLE users"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region; DROP TABLE users` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("neutralizes a backtick in a dimension by doubling it", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region`"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region``` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); }); - test("throws for a dimension with a backtick / quote", () => { + test("throws for a dimension containing a control character", () => { expect(() => buildMetricSql(registration, { measures: ["arr"], - dimensions: ["region`"], + dimensions: ["region\tbad"], }), - ).toThrow(/not a valid identifier/); + ).toThrow(/not a valid identifier|control character/); }); }); @@ -447,7 +479,7 @@ describe("analytics metric route (Phase 1)", () => { expect.anything(), expect.objectContaining({ statement: - "SELECT MEASURE(arr) AS arr FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", warehouse_id: "test-warehouse-id", }), expect.any(AbortSignal), @@ -766,6 +798,7 @@ describe("loadMetricRegistry", () => { }); afterEach(() => { + __resetMetricRegistryCache(); rmSync(dir, { recursive: true, force: true }); }); @@ -837,6 +870,53 @@ describe("loadMetricRegistry", () => { ); }); + test("rejects more than 200 metric views (runtime parity with typegen cap)", async () => { + // A config that fails type generation (MAX_METRIC_VIEWS) must not silently + // pass at runtime. z.record has no `.max` in zod 4, so this is enforced by + // the schema's superRefine. + const metricViews: Record = {}; + for (let i = 0; i < 201; i++) { + metricViews[`m_${i}`] = { source: `cat.sch.view_${i}` }; + } + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("rejects an FQN segment longer than 255 chars (per-segment cap)", async () => { + // The per-segment length cap is not expressible as a whole-string + // maxLength; the schema's superRefine enforces it so runtime matches the + // typegen resolver. + const longSegment = "a".repeat(256); + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: `cat.sch.${longSegment}` } }, + }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("accepts exactly 200 views and a 255-char segment (boundary)", async () => { + const metricViews: Record = {}; + for (let i = 0; i < 200; i++) { + metricViews[`m_${i}`] = { source: `cat.sch.view_${i}` }; + } + // One entry with a segment at exactly the 255 limit — must pass. + metricViews.m_0 = { source: `cat.sch.${"a".repeat(255)}` }; + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + await expect(loadMetricRegistry(dir)).resolves.toBeDefined(); + }); + test("re-reads only after the file changes (mtime-validated cache)", async () => { // getMetricRegistry caches by dir and revalidates via stat; a second call // with no change returns the same object, and a rewrite is picked up. @@ -863,6 +943,51 @@ describe("loadMetricRegistry", () => { expect(third).not.toBe(first); expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); }); + + test("picks up a SAME-SIZE edit (ctime invalidation, not just size)", async () => { + // The failure mode size+mtime alone would miss: rewrite the config to the + // EXACT same byte length (repoint `source` to an equal-length FQN). `size` + // is unchanged and a coarse-mtime FS might not advance `mtimeMs`, but + // `ctimeMs` bumps on any write — so the new source must be served. + const p = path.join(dir, "metric-views.json"); + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, + }), + ); + const before = await getMetricRegistry(dir); + expect(before.revenue?.source).toBe("cat.sch.rev_aaa"); + + // Same-length replacement: "rev_aaa" → "rev_bbb" keeps the file byte-count + // identical, so `size` cannot disambiguate. + const sizeBefore = statSync(p).size; + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_bbb" } }, + }), + ); + expect(statSync(p).size).toBe(sizeBefore); // same size, as designed + + const after = await getMetricRegistry(dir); + expect(after.revenue?.source).toBe("cat.sch.rev_bbb"); + }); + + test("__resetMetricRegistryCache forces a cold re-read", async () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + const first = await getMetricRegistry(dir); + __resetMetricRegistryCache(); + const second = await getMetricRegistry(dir); + // Cleared cache → fresh parse → a new object (not the memoized instance). + expect(second).not.toBe(first); + expect(Object.keys(second)).toEqual(["revenue"]); + }); }); // ── Phase 2: the structured filter engine (translator + validator). @@ -893,7 +1018,7 @@ describe("metric — filter translator", () => { operator: "equals", values: ["EMEA"], }); - expect(where).toBe("region = :f_0"); + expect(where).toBe("`region` = :f_0"); expect(parameters).toEqual({ f_0: { __sql_type: "STRING", value: "EMEA" }, }); @@ -905,7 +1030,7 @@ describe("metric — filter translator", () => { operator: "notEquals", values: ["EMEA"], }); - expect(where).toBe("region <> :f_0"); + expect(where).toBe("`region` <> :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); }); @@ -915,7 +1040,7 @@ describe("metric — filter translator", () => { operator: "in", values: ["EMEA", "APAC", "AMER"], }); - expect(where).toBe("region IN (:f_0, :f_1, :f_2)"); + expect(where).toBe("`region` IN (:f_0, :f_1, :f_2)"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); expect(parameters.f_1).toEqual({ __sql_type: "STRING", value: "APAC" }); expect(parameters.f_2).toEqual({ __sql_type: "STRING", value: "AMER" }); @@ -927,7 +1052,7 @@ describe("metric — filter translator", () => { operator: "notIn", values: ["EMEA", "APAC"], }); - expect(where).toBe("region NOT IN (:f_0, :f_1)"); + expect(where).toBe("`region` NOT IN (:f_0, :f_1)"); expect(Object.keys(parameters)).toHaveLength(2); }); @@ -937,7 +1062,7 @@ describe("metric — filter translator", () => { operator: "gt", values: [10000], }); - expect(where).toBe("deal_size > :f_0"); + expect(where).toBe("`deal_size` > :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "INT", value: "10000" }); }); @@ -947,7 +1072,7 @@ describe("metric — filter translator", () => { operator: "gte", values: [5000], }); - expect(where).toBe("deal_size >= :f_0"); + expect(where).toBe("`deal_size` >= :f_0"); }); test("lt → ` < :f_0`", () => { @@ -956,7 +1081,7 @@ describe("metric — filter translator", () => { operator: "lt", values: [100], }); - expect(where).toBe("deal_size < :f_0"); + expect(where).toBe("`deal_size` < :f_0"); }); test("lte → ` <= :f_0`", () => { @@ -965,7 +1090,7 @@ describe("metric — filter translator", () => { operator: "lte", values: [50000], }); - expect(where).toBe("deal_size <= :f_0"); + expect(where).toBe("`deal_size` <= :f_0"); }); test("contains → ` LIKE :f_0` (value wrapped in %...%)", () => { @@ -974,7 +1099,7 @@ describe("metric — filter translator", () => { operator: "contains", values: ["MEA"], }); - expect(where).toBe("region LIKE :f_0"); + expect(where).toBe("`region` LIKE :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%MEA%" }); }); @@ -984,7 +1109,7 @@ describe("metric — filter translator", () => { operator: "notContains", values: ["test"], }); - expect(where).toBe("region NOT LIKE :f_0"); + expect(where).toBe("`region` NOT LIKE :f_0"); expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%test%", @@ -996,7 +1121,7 @@ describe("metric — filter translator", () => { member: "region", operator: "set", }); - expect(where).toBe("region IS NOT NULL"); + expect(where).toBe("`region` IS NOT NULL"); expect(parameters).toEqual({}); }); @@ -1005,7 +1130,7 @@ describe("metric — filter translator", () => { member: "region", operator: "notSet", }); - expect(where).toBe("region IS NULL"); + expect(where).toBe("`region` IS NULL"); expect(parameters).toEqual({}); }); }); @@ -1018,7 +1143,7 @@ describe("metric — filter translator", () => { { member: "segment", operator: "equals", values: ["Enterprise"] }, ], }); - expect(where).toBe("(region = :f_0 AND segment = :f_1)"); + expect(where).toBe("(`region` = :f_0 AND `segment` = :f_1)"); expect(parameters.f_0.value).toBe("EMEA"); expect(parameters.f_1.value).toBe("Enterprise"); }); @@ -1030,7 +1155,7 @@ describe("metric — filter translator", () => { { member: "region", operator: "equals", values: ["APAC"] }, ], }); - expect(where).toBe("(region = :f_0 OR region = :f_1)"); + expect(where).toBe("(`region` = :f_0 OR `region` = :f_1)"); }); test("AND-of-OR composes nested groups", () => { @@ -1045,7 +1170,7 @@ describe("metric — filter translator", () => { }, ], }); - expect(where).toContain("(region IN ("); + expect(where).toContain("(`region` IN ("); expect(where).toContain(" AND "); expect(where).toContain("OR"); }); @@ -1219,14 +1344,25 @@ describe("metric — filter translator", () => { expect(Object.keys(parameters)).toHaveLength(3); }); - test("a filter member failing DIMENSION_NAME_PATTERN throws before SQL is built", () => { + test("an injection-shaped filter member is neutralized by quoting", () => { + // No control char → a valid (if nonexistent) column name: quoted whole, + // the `;`/`DROP`/`--` are inert inside the backtick-delimited identifier. + const { where } = render({ + member: "region; DROP TABLE foo --", + operator: "equals", + values: ["x"], + }); + expect(where).toBe("`region; DROP TABLE foo --` = :f_0"); + }); + + test("a filter member with a control character throws before SQL is built", () => { expect(() => render({ - member: "region; DROP TABLE foo --", + member: "region\ndrop", operator: "equals", values: ["x"], }), - ).toThrowError(/not a valid identifier/); + ).toThrowError(/not a valid identifier|control character/); }); }); @@ -1383,7 +1519,7 @@ describe("metric — filter translator", () => { operator: "equals", values: ["x"], }); - expect(where).toBe("ghost_column = :f_0"); + expect(where).toBe("`ghost_column` = :f_0"); }); }); @@ -1407,7 +1543,7 @@ describe("metric — filter translator", () => { }); test("rejects a name that is BOTH a measure and a dimension", () => { - // The corruption case: `SELECT MEASURE(x) AS x, x ... GROUP BY ALL` + // The corruption case: `SELECT MEASURE(`x`) AS `x`, x ... GROUP BY ALL` // materializes two `x` columns and the second overwrites the first. expect(() => validateMetricRequest({ @@ -1613,7 +1749,7 @@ describe("metric — filter translator", () => { expect(executeMock.mock.calls[0][1]).toEqual( expect.objectContaining({ statement: - "SELECT MEASURE(ghost_measure) AS ghost_measure FROM `cat`.`sch`.`revenue_metrics`", + "SELECT MEASURE(`ghost_measure`) AS `ghost_measure` FROM `cat`.`sch`.`revenue_metrics`", }), ); diff --git a/packages/shared/src/schemas/metric-fqn.ts b/packages/shared/src/schemas/metric-fqn.ts index 98fca3d5..be96c162 100644 --- a/packages/shared/src/schemas/metric-fqn.ts +++ b/packages/shared/src/schemas/metric-fqn.ts @@ -140,21 +140,54 @@ export function isValidFqn(fqn: string): boolean { * @throws If any segment contains a control character or newline. */ export function quoteFqnForSql(fqn: string): string { - // Reject anything that cannot be represented inside a backtick-quoted - // identifier. \p{Cc} is the Unicode "control" category, which covers C0 - // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. - const CONTROL_OR_NEWLINE = /\p{Cc}/u; - return fqn - .split(".") - .map((segment) => { - if (CONTROL_OR_NEWLINE.test(segment)) { - throw new Error( - `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, - ); - } - // Double every backtick — the only break-out from a backtick-quoted - // identifier — then wrap the whole segment in backticks. - return `\`${segment.replace(/`/g, "``")}\``; - }) - .join("."); + return fqn.split(".").map(quoteIdentifier).join("."); +} + +/** + * The Unicode "control" category (`\p{Cc}`): C0 (incl. `\n`, `\r`, `\t`), DEL, + * and C1 — every control character/newline. These have no valid escape inside + * a backtick-quoted identifier, so a name containing one cannot be safely + * quoted and is rejected. + */ +const CONTROL_OR_NEWLINE = /\p{Cc}/u; + +/** + * Is `name` a column/measure/dimension identifier that {@link quoteIdentifier} + * can safely escape? True for any non-empty string free of control characters + * and newlines. + * + * This is the **column-identifier** grammar — deliberately broader than the + * FQN-segment grammar ({@link UC_FQN_PATTERN}, which also forbids `.` and `/` + * because they are FQN structural characters). A metric view's measure or + * dimension is a single *delimited* column identifier: once backtick-quoted it + * may legally contain dots, slashes, spaces, hyphens, and non-ASCII — anything + * but a control character. The type-generator emits DESCRIBE column names + * verbatim into the generated `measureKeys`/`dimensionKeys` unions, so the + * runtime must accept exactly what can be safely quoted, or a generated name + * would typecheck but fail at runtime. + */ +export function isValidColumnName(name: string): boolean { + return name.length > 0 && !CONTROL_OR_NEWLINE.test(name); +} + +/** + * Quote a SINGLE identifier (one column/measure/dimension name, or one FQN + * segment) as a backtick-delimited identifier for safe SQL interpolation. + * + * Unlike {@link quoteFqnForSql}, this does NOT split on `.` — the whole input + * is one identifier, so a column literally named `net.revenue` becomes + * `` `net.revenue` `` (one identifier), not `` `net`.`revenue` `` (two). The + * backtick — the only break-out character — is doubled; control characters and + * newlines have no valid escape and are rejected. + * + * @throws If `name` contains a control character or newline. + */ +export function quoteIdentifier(name: string): string { + if (CONTROL_OR_NEWLINE.test(name)) { + throw new Error( + `Cannot quote identifier "${name}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, + ); + } + // Double every backtick, then wrap in backticks. + return `\`${name.replace(/`/g, "``")}\``; } diff --git a/packages/shared/src/schemas/metric-source.ts b/packages/shared/src/schemas/metric-source.ts index 2e4e1dfd..4d64458a 100644 --- a/packages/shared/src/schemas/metric-source.ts +++ b/packages/shared/src/schemas/metric-source.ts @@ -18,7 +18,25 @@ */ import { z } from "zod"; -import { UC_FQN_PATTERN } from "./metric-fqn"; +import { MAX_UC_OBJECT_NAME_LENGTH, UC_FQN_PATTERN } from "./metric-fqn"; + +/** + * Safety cap on the number of declared metric views — a typo / DoS guard, NOT + * a Unity Catalog limit. Mirrors the type-generator's `MAX_METRIC_VIEWS` so + * runtime config validation and type generation accept exactly the same + * configs (a config that fails generation must not silently pass at runtime). + */ +const MAX_METRIC_VIEWS = 200; + +/** + * Whole-FQN length cap: three max-length UC object names plus the two dots. + * The per-segment cap ({@link MAX_UC_OBJECT_NAME_LENGTH}) is enforced in the + * `superRefine` below; this whole-string bound is the declarative half (it + * serializes to a JSON-schema `maxLength`, whereas the per-segment check — like + * the entry-count cap — cannot be expressed declaratively and lives in the + * refinement, so it is a runtime/type-generator gate only). + */ +const MAX_FQN_LENGTH = MAX_UC_OBJECT_NAME_LENGTH * 3 + 2; /** * Three-part Unity Catalog FQN matcher, composed from the single-segment @@ -62,6 +80,7 @@ export const metricEntrySchema = z source: z .string() .regex(UC_THREE_PART_FQN_PATTERN) + .max(MAX_FQN_LENGTH) .describe( "Three-part Unity Catalog FQN of the metric view: ..", ) @@ -94,7 +113,39 @@ export const metricSourceSchema = z .strict() .describe( "Schema for AppKit metric-views.json — declares Unity Catalog Metric View sources for the analytics plugin's metric-view path. Each entry under 'metricViews' binds a metric key to a UC metric view FQN and an executor ('app_service_principal' shared cache, or 'user' per-user cache). Object form (rather than bare string) at v1 enables future per-entry option growth without breaking changes.", - ); + ) + // Caps that cannot be expressed declaratively (zod 4's `z.record` has no + // `.max`, and a per-dot-segment length bound isn't a whole-string + // `maxLength`). Enforced here so runtime config validation matches the + // type-generator's `resolveMetricConfig` exactly — a config that fails type + // generation must not silently pass at runtime. These refinements are + // invisible to `z.toJSONSchema`, so the generated JSON schema carries only + // the declarative `maxLength` on `source`; runtime + type-generator remain + // the authoritative gates for the entry-count and per-segment caps. + .superRefine((value, ctx) => { + const entries = value.metricViews ? Object.entries(value.metricViews) : []; + + if (entries.length > MAX_METRIC_VIEWS) { + ctx.addIssue({ + code: "custom", + message: `too many metric views: ${entries.length} declared, exceeding the maximum of ${MAX_METRIC_VIEWS}`, + path: ["metricViews"], + }); + } + + for (const [key, entry] of entries) { + const segments = entry.source.split("."); + for (let i = 0; i < segments.length; i++) { + if (segments[i].length > MAX_UC_OBJECT_NAME_LENGTH) { + ctx.addIssue({ + code: "custom", + message: `metric source segment ${i + 1} is ${segments[i].length} characters, exceeding the per-segment maximum of ${MAX_UC_OBJECT_NAME_LENGTH}`, + path: ["metricViews", key, "source"], + }); + } + } + } + }); export type MetricKey = z.infer; export type MetricExecutor = z.infer; From 2670a0208b287c47ec487a6c7c8e977594db5a10 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Fri, 10 Jul 2026 14:36:47 +0200 Subject: [PATCH 9/9] refactor(appkit): split metric runtime modules Signed-off-by: Atila Fassina --- .../shared/appkit-types/metric-views.d.ts | 19 + .../appkit/src/plugins/analytics/metric.ts | 1393 +---------------- .../appkit/src/plugins/analytics/mv/cache.ts | 71 + .../src/plugins/analytics/mv/constants.ts | 122 ++ .../src/plugins/analytics/mv/formatters.ts | 319 ++++ .../appkit/src/plugins/analytics/mv/index.ts | 9 + .../src/plugins/analytics/mv/registry.ts | 192 +++ .../src/plugins/analytics/mv/schemas.ts | 356 +++++ .../appkit/src/plugins/analytics/mv/types.ts | 25 + 9 files changed, 1114 insertions(+), 1392 deletions(-) create mode 100644 packages/appkit/src/plugins/analytics/mv/cache.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/constants.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/formatters.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/index.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/registry.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/schemas.ts create mode 100644 packages/appkit/src/plugins/analytics/mv/types.ts diff --git a/apps/dev-playground/shared/appkit-types/metric-views.d.ts b/apps/dev-playground/shared/appkit-types/metric-views.d.ts index e0ada374..1c7fb87a 100644 --- a/apps/dev-playground/shared/appkit-types/metric-views.d.ts +++ b/apps/dev-playground/shared/appkit-types/metric-views.d.ts @@ -30,23 +30,31 @@ declare module "@databricks/appkit-ui/react" { measures: { "active_accounts": { type: "bigint"; + display_name: "Active Accounts"; + format: "#,##0"; }; "churn_rate": { type: "decimal"; + display_name: "Churn Rate"; }; "avg_ltv": { type: "double"; + display_name: "Average LTV"; + format: "$#,##0.00"; }; }; dimensions: { "segment": { type: "string"; + display_name: "Customer Segment"; }; "region": { type: "string"; + display_name: "Region"; }; "csm_email": { type: "string"; + display_name: "CSM Email"; }; }; }; @@ -80,27 +88,38 @@ declare module "@databricks/appkit-ui/react" { measures: { "mrr": { type: "double"; + display_name: "Monthly Recurring Revenue"; + format: "$#,##0.00"; }; "arr": { type: "double"; + display_name: "Annual Recurring Revenue"; + format: "$#,##0.00"; description: "Annualized contract value across all active subscriptions"; }; "new_arr": { type: "double"; + display_name: "New ARR"; + format: "$#,##0.00"; }; "churned_arr": { type: "double"; + display_name: "Churned ARR"; + format: "$#,##0.00"; }; }; dimensions: { "region": { type: "string"; + display_name: "Region"; }; "segment": { type: "string"; + display_name: "Customer Segment"; }; "created_at": { type: "timestamp_ltz"; + display_name: "Subscription Start"; time_grain: readonly ["day", "hour", "minute", "month", "quarter", "week", "year"]; }; }; diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts index aabfdc89..3e8a6f47 100644 --- a/packages/appkit/src/plugins/analytics/metric.ts +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -1,1392 +1 @@ -import { createHash } from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; -import { z } from "zod"; -// Canonical metric-source schema — the single source of truth for -// `metric-views.json`. Imported from the shared source directly (matching the -// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the -// same tree) so the runtime and the generated JSON schema validate identically. -import { - isValidColumnName, - isValidFqn, - quoteFqnForSql, - quoteIdentifier, -} from "../../../../shared/src/schemas/metric-fqn"; -import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; -import { AuthenticationError, ValidationError } from "../../errors"; -import { createLogger } from "../../logging/logger"; -import type { - IAnalyticsMetricRequest, - MetricFilter, - MetricFilterOperatorName, - MetricLane, - MetricPredicate, - MetricRegistration, -} from "./types"; -import { normalizeAnalyticsFormat } from "./types"; - -const logger = createLogger("analytics:metric"); - -/** - * Default queries directory. Mirrors `AppManager`'s - * `path.resolve(process.cwd(), "config/queries")` so dev mode and production - * share a single source of truth for where metric config lives. Exported so - * `AnalyticsPlugin` can default `config.queriesDir` to the same path. - */ -export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); -const METRIC_CONFIG_FILE = "metric-views.json"; - -/** - * Measure, dimension, and filter-member names are **column identifiers**: they - * are validated by the shared {@link isValidColumnName} (rejects only control - * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at - * every interpolation point. Quoting — not a narrow ASCII allowlist — is the - * injection boundary, so the runtime accepts the full delimited-identifier - * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). - * There is deliberately NO name allowlist: a well-formed-but-unknown column - * falls through to the warehouse and surfaces as a sanitized canonical error. - * - * Time-grain token shape. Unlike the column identifiers above, the grain is - * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, - * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a - * narrow keyword-shaped gate — that pattern is what keeps a hostile token out - * of the quoted-literal position. - */ -const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; - -/** - * The exact twelve filter operators allowed at v1. The runtime tuple is the - * server-side source of truth; the client-side type union - * `MetricFilterOperatorName` mirrors these names statically. - */ -const METRIC_FILTER_OPERATORS = [ - "equals", - "notEquals", - "in", - "notIn", - "gt", - "gte", - "lt", - "lte", - "contains", - "notContains", - "set", - "notSet", -] as const satisfies readonly MetricFilterOperatorName[]; - -/** - * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — - * enough for any real BI filter UI, low enough that a hostile or malformed - * payload cannot stack-overflow the recursive validator or translator. - * - * The depth count is the number of nested `{ and }` / `{ or }` wrappers - * encountered while descending — leaf predicates do not count toward depth. - */ -const METRIC_FILTER_MAX_DEPTH = 8; - -/** - * Cardinality caps on user-controlled arrays. Closes the recurring - * `unbounded-request-parameters` finding: a hostile caller could otherwise - * send `values: [...10M items...]` and exhaust the validator + the named - * bind-var binding step. The limits below are deliberately generous — higher - * than any real BI UI would emit — so legitimate traffic never trips them. - */ -const METRIC_MEASURES_MAX = 50; -const METRIC_DIMENSIONS_MAX = 20; -const METRIC_FILTER_VALUES_MAX = 1000; -const METRIC_LIMIT_MAX = 100_000; - -/** - * Maximum number of children per AND/OR group node. Without this cap a single - * flat group like `{ and: [...10M empty objects...] }` would push tens of - * millions of frames onto the iterative pre-check's stack — OOM before - * validation even gets to Zod. The Zod schema enforces the same cap so the - * rejection point is consistent regardless of which validator catches it - * first. - */ -const METRIC_FILTER_GROUP_MAX = 100; - -/** Operators that require exactly one value. */ -const SINGLE_VALUE_OPERATORS = new Set([ - "equals", - "notEquals", - "gt", - "gte", - "lt", - "lte", - "contains", - "notContains", -]); - -/** Operators that require at least one value. */ -const LIST_VALUE_OPERATORS = new Set(["in", "notIn"]); - -/** Operators that reject `values` entirely. */ -const NULL_OPERATORS = new Set(["set", "notSet"]); - -/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ -const STRING_OPERATORS = new Set([ - "contains", - "notContains", -]); - -/** - * Map an entry's declared `executor` to the internal execution lane: - * - `"user"` → `"obo"` (per-user cache, on-behalf-of) - * - `"app_service_principal"` (default) → `"sp"` (shared cache) - */ -function laneFromExecutor( - executor: "app_service_principal" | "user", -): MetricLane { - return executor === "user" ? "obo" : "sp"; -} - -/** - * Read and validate `config/queries/metric-views.json` into a metric registry. - * - * Async and stateless — registration is a pure config parse with no warehouse - * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single - * `metricViews` map makes keys unique by construction, so there is no - * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps - * the event loop free: metric views are already heavier than a plain `.sql` - * query on the warehouse side, so the SDK layer must not add a blocking read - * on top. Caching + mtime-revalidation is layered on top by - * {@link getMetricRegistry} — this function always hits disk. - * - * Returns an empty registry when the file is absent: the metric-view path is - * additive and dormant until an app opts in by adding the config. A malformed - * file (unreadable, invalid JSON, or schema violation) throws — the caller - * surfaces a 503 rather than masking a broken deployment as a 404 for every - * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the - * file heals on the next request. - */ -export async function loadMetricRegistry( - queriesDir: string = QUERIES_DIR, -): Promise> { - const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); - - let raw: string; - try { - raw = await fs.readFile(metricPath, "utf8"); - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - return Object.create(null); - } - throw err; - } - - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err) { - throw new Error( - `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, - ); - } - - const result = metricSourceSchema.safeParse(parsed); - if (!result.success) { - const issues = result.error.issues - .map((i) => `${i.path.join(".")}: ${i.message}`) - .join("; "); - throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); - } - - // Null-prototype map so a metric key that collides with an inherited - // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) - // cannot resolve to a truthy non-registration at the `registry[key]` read - // site and slip past the unknown-key 404. Keys are still grammar-gated by - // `metricKeySchema` (identifier shape), but the null prototype removes the - // whole class of inherited-property lookups as a boundary. - const registry: Record = Object.create(null); - for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { - registry[key] = { - key, - source: entry.source, - lane: laneFromExecutor(entry.executor), - }; - } - - logger.debug( - "Loaded metric registry: %d entry(ies)", - Object.keys(registry).length, - ); - return registry; -} - -/** - * Signature of the config file the cached registry was parsed from: its - * change time, modification time, and size — all from one `stat`, no read. - * - * `ctimeMs` (inode change time) is the key guard against a stale serve: it - * bumps on ANY metadata or content change and, unlike `mtimeMs`, cannot be - * restored to a prior value by tooling (`utimes` sets atime/mtime but not - * ctime). So an edit that preserves both size and mtime — a same-length value - * swap (e.g. repointing `source` to an equal-length FQN, or flipping - * `executor` between two equal-length values) on a coarse-mtime filesystem — - * still changes ctime and invalidates the cache. Without it, such an edit - * could serve a stale `source`/`lane`, reintroducing the exact stale-serve - * class the cache-key `source` salt was added to prevent, one layer up. - * A content hash would be stronger still but requires reading the file, which - * is the cost this cache exists to avoid. - */ -interface RegistryCacheSignature { - ctimeMs: number; - mtimeMs: number; - size: number; -} - -/** - * Module-level registry cache, keyed by the resolved queries directory. - * - * Keyed by DIR (not by plugin instance) because the registry is a pure - * function of the config file at that path — warehouse-independent, and two - * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see - * the same registry. Instance state would parse the identical file twice and - * risk divergence; a dir-keyed module cache shares one parse. - */ -const metricRegistryCache = new Map< - string, - { - signature: RegistryCacheSignature; - registry: Record; - } ->(); - -/** - * Clear the module-level registry cache. - * - * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the - * process; production never needs to clear it (a changed file is picked up via - * the stat signature). Tests that reuse a directory — or assert cold-load - * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional - * rather than relying on each test happening to use a unique temp dir. - */ -export function __resetMetricRegistryCache(): void { - metricRegistryCache.clear(); -} - -/** - * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only - * when `metric-views.json` has changed since the cached copy. - * - * Behaves like the sibling `.sql` query path (which re-reads per request) but - * cheaper: the steady-state cost is a single async `stat`, and the read + - * `JSON.parse` + zod validation are skipped when the file's - * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed - * semantics without the old permanent memo: - * - * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next - * request re-parses and serves the new registry with no server restart. - * - **Self-heal** — a load failure is NOT cached (we only populate the cache - * on a successful parse), so a fixed config is picked up on the next - * request instead of latching a 503 forever. - * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding - * the file later is picked up on the next request. - * - * Concurrency: two simultaneous cold requests may both `stat`+read before - * either populates the cache. That is a harmless redundant read of the same - * file (the sibling `.sql` path does not single-flight either), so no lock is - * taken. - * - * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so - * the route can surface a 503; the cache is left untouched on failure. - */ -export async function getMetricRegistry( - queriesDir: string = QUERIES_DIR, -): Promise> { - const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); - - let signature: RegistryCacheSignature | null; - try { - const stats = await fs.stat(metricPath); - signature = { - ctimeMs: stats.ctimeMs, - mtimeMs: stats.mtimeMs, - size: stats.size, - }; - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - // Absent file → dormant. Drop any stale cache entry (the file may have - // been deleted) and return an empty registry without touching the cache. - metricRegistryCache.delete(queriesDir); - signature = null; - } else { - // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal - // for THIS request → the route surfaces a 503, consistent with the - // malformed-config → 503 path. It is not latched: with the self-heal - // design a transient error clears on the next request, which is strictly - // better than the old memo that could latch-and-serve-stale. - throw err; - } - } - - if (signature === null) { - return Object.create(null); - } - - const cached = metricRegistryCache.get(queriesDir); - if ( - cached !== undefined && - cached.signature.ctimeMs === signature.ctimeMs && - cached.signature.mtimeMs === signature.mtimeMs && - cached.signature.size === signature.size - ) { - return cached.registry; - } - - // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed - // file never latches — the next request re-attempts and heals. - const registry = await loadMetricRegistry(queriesDir); - metricRegistryCache.set(queriesDir, { signature, registry }); - return registry; -} - -/** - * Validate a metric-view FQN and return it backtick-quoted for interpolation. - * - * The FQN ships in the SQL string — it cannot be parameterized — so it passes - * two shared, zod-free gates at construction time: - * - * 1. {@link isValidFqn} — the three-part UC grammar (exactly three segments, - * each matching `UC_FQN_PATTERN`). This is the SAME predicate the - * type-generator's describe seam uses and is derived from the same - * per-segment charset as the canonical Zod schema, so config-time, - * generation-time, and runtime accept exactly the same names. - * 2. {@link quoteFqnForSql} — backtick-quotes each segment (doubling embedded - * backticks) so the FQN cannot break out of its identifier position. This - * is the injection boundary; it is why the runtime can accept the full UC - * quoted-identifier grammar (hyphens, non-ASCII) rather than the narrow - * ASCII allowlist it used before. - * - * The registry loader already enforces the grammar via `metricSourceSchema`; - * this re-gate is defense-in-depth for any code path that reaches - * `buildMetricSql` without going through a parsed registry (it is exported), - * matching how measures, dimensions, and the grain are each re-gated at their - * interpolation points. - */ -function quoteSafeFqn(fqn: string): string { - if (!isValidFqn(fqn)) { - throw new Error( - `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, - ); - } - return quoteFqnForSql(fqn); -} - -/** - * Structural validation schema for the metric request body. - * - * The schema is **static** — any grammar-valid measure/dimension identifier is - * accepted; it is NOT a dynamic per-key `z.enum(knownMeasures)`. Unknown-but- - * well-formed names are not rejected here; they reach the warehouse and surface - * as a sanitized canonical error. The identifier grammar gate is enforced in - * {@link buildMetricSql}/{@link renderFilter} at interpolation time; the body - * schema stays structural (item shape, cardinality caps, operator enum, - * per-operator value cardinality, and AND/OR depth). - * - * `filter` is recursive (`Predicate | { and: [...] } | { or: [...] }`), built - * with `z.lazy`. `timeGrain` buckets `timeDimension` via `date_trunc` (see - * {@link renderDimensionClause}); the two cross-field rules (grain requires - * timeDimension; timeDimension must be one of dimensions) live in the - * `superRefine` below. - */ - -/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ -const filterPredicateSchema: z.ZodType = z - .object({ - member: z - .string() - .min(1, { message: "filter predicate 'member' cannot be empty" }) - .refine(isValidColumnName, { - message: - "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", - }), - operator: z.string().min(1, { - message: "filter predicate 'operator' cannot be empty", - }) as z.ZodType, - values: z - .array(z.union([z.string(), z.number()])) - .max(METRIC_FILTER_VALUES_MAX, { - message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, - }) - .optional(), - }) - .strict(); - -/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ -const filterSchema: z.ZodType = z.lazy(() => - z.union([ - filterPredicateSchema, - z - .object({ - and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { - message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, - }), - }) - .strict(), - z - .object({ - or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { - message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, - }), - }) - .strict(), - ]), -); - -const metricRequestSchema = z - .object({ - measures: z - .array( - z - .string() - .min(1, "measure name cannot be empty") - .refine(isValidColumnName, { - message: - "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", - }), - ) - .min(1, "at least one measure is required") - .max(METRIC_MEASURES_MAX, { - message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, - }), - dimensions: z - .array( - z - .string() - .min(1, "dimension name cannot be empty") - .refine(isValidColumnName, { - message: - "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", - }), - ) - .max(METRIC_DIMENSIONS_MAX, { - message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, - }) - .optional(), - filter: filterSchema.optional(), - // Grammar-shaped bucketing grain, applied to `timeDimension` via - // `date_trunc`. The token is validated for safety here; the grain literal - // is interpolated (single-quoted, never a bind param) in - // `renderDimensionClause`, so this pattern gate is the security boundary. - timeGrain: z - .string() - .min(1, { message: "timeGrain cannot be empty" }) - .regex(TIME_GRAIN_PATTERN, { - message: "timeGrain must match /^[a-z][a-z_]*$/", - }) - .optional(), - // The single dimension `timeGrain` applies to via `date_trunc`. A column - // identifier (backtick-quoted at interpolation), so it accepts the full - // delimited-identifier grammar. Cross-field rules in `superRefine`: - // required when `timeGrain` is set, and must be one of `dimensions`. - timeDimension: z - .string() - .min(1, { message: "timeDimension cannot be empty" }) - .refine(isValidColumnName, { - message: - "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", - }) - .optional(), - limit: z - .number() - .int({ message: "limit must be an integer" }) - .positive({ message: "limit must be positive" }) - .max(METRIC_LIMIT_MAX, { - message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, - }) - .optional(), - format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), - }) - .strict() - .superRefine((value, ctx) => { - if (value.filter != null) { - validateFilterTree(value.filter, ctx, ["filter"], 0); - } - - // Uniqueness. Measures and dimensions become SELECT columns aliased to - // their own name (`MEASURE(x) AS x`, `x`); the warehouse returns one row - // object keyed by column name, so a repeated name — a duplicate measure, a - // duplicate dimension, or a name appearing as BOTH a measure and a - // dimension — collapses to a single key and silently drops a value during - // row materialization. Reject the collision here rather than emit SQL that - // corrupts rows. (The grammar gate at build time is a safety boundary, not - // a uniqueness check, so this lives in validation.) - const seen = new Set(); - const collided = new Set(); - for (const name of [...value.measures, ...(value.dimensions ?? [])]) { - if (seen.has(name)) { - collided.add(name); - } - seen.add(name); - } - if (collided.size > 0) { - ctx.addIssue({ - code: "custom", - message: - "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", - path: ["measures"], - }); - } - - // Delivery format. The metric route delivers JSON rows only at v1 (the - // cache salts on a fixed "JSON_ARRAY" and the route always routes through - // the JSON path). Accepting an Arrow format would silently return JSON — - // reject it loudly until Arrow parity ships. Legacy aliases normalize - // first (`JSON`→`JSON_ARRAY`, `ARROW`→`ARROW_STREAM`). - if ( - value.format != null && - normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" - ) { - ctx.addIssue({ - code: "custom", - message: - "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", - path: ["format"], - }); - } - - // Cross-field grain rules. `timeGrain` buckets exactly one selected - // dimension via `date_trunc`, so it requires an explicit `timeDimension`, - // and that dimension must be selected (so it is in the SELECT list and - // `GROUP BY ALL`). - if (value.timeGrain != null && value.timeDimension == null) { - ctx.addIssue({ - code: "custom", - message: "timeDimension is required when timeGrain is set", - path: ["timeDimension"], - }); - } - if ( - value.timeDimension != null && - !(value.dimensions ?? []).includes(value.timeDimension) - ) { - ctx.addIssue({ - code: "custom", - message: "timeDimension must be one of dimensions", - path: ["timeDimension"], - }); - } - }) as z.ZodType; - -/** - * Recursive Zod-time validator for the filter tree. - * - * Pushes structured issues into the refinement context with stable paths - * (`filter.and.0.or.2.member`, etc.) so the canonical 400 error carries - * actionable diagnostics. Keeps the registry-free concerns in one descent: - * - * 1. Operator is one of the twelve. - * 2. Per-operator value cardinality (single / list / null operators). - * 3. `contains`/`notContains` carry a string value. - * 4. AND/OR nesting depth cap. - * - * There is NO member-allowlist and NO op⇄dimension-type check — the member is - * grammar-gated at SQL-build time, and dimension types are not tracked (no - * registry metadata). Returns void; issues accumulate on `ctx`. - */ -function validateFilterTree( - node: MetricFilter, - ctx: z.RefinementCtx, - path: Array, - depth: number, -): void { - if (node === null || typeof node !== "object") { - // The base schema rejects this via the union, but stay defensive. - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path, - message: "filter node must be a Predicate or { and } / { or } group", - }); - return; - } - - if ("and" in node || "or" in node) { - if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path, - message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, - }); - return; - } - - const groupKey = "and" in node ? "and" : "or"; - const children = ( - node as { and?: ReadonlyArray } & { - or?: ReadonlyArray; - } - )[groupKey]; - - if (!Array.isArray(children)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, groupKey], - message: `filter ${groupKey} group must be an array of predicates or nested groups`, - }); - return; - } - - // Reject empty `or` groups: an empty disjunction is vacuously false, which - // silently drops the surrounding intent. Empty `and` is OK (vacuously true - // → no constraint). Forcing the caller to omit the predicate entirely is - // the only unambiguous choice. - if (groupKey === "or" && children.length === 0) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "or"], - message: "filter 'or' group must contain at least one predicate", - }); - return; - } - - children.forEach((child, idx) => { - validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); - }); - return; - } - - // Leaf predicate. The base schema already enforced shape; here we layer in - // the operator vocabulary + per-operator value cardinality. - const predicate = node as MetricPredicate; - - if ( - !METRIC_FILTER_OPERATORS.includes( - predicate.operator as MetricFilterOperatorName, - ) - ) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "operator"], - message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, - }); - // No further checks meaningful when the operator is unknown. - return; - } - - const op = predicate.operator; - const values = predicate.values; - const valuesLen = values?.length ?? 0; - - if (NULL_OPERATORS.has(op)) { - if (values != null && valuesLen > 0) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" must not carry values`, - }); - } - } else if (SINGLE_VALUE_OPERATORS.has(op)) { - if (valuesLen !== 1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, - }); - } - } else if (LIST_VALUE_OPERATORS.has(op)) { - if (valuesLen < 1) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" requires at least one value`, - }); - } - } - - // Op⇄value-type: string operators require a string value. This is a - // registry-free structural check (not an op⇄dimension-type check) — it - // converts what would otherwise be a synchronous throw in `renderPredicate` - // (rendered as a 500) into a clean 400 at validation time. - if (STRING_OPERATORS.has(op) && valuesLen > 0) { - const v = predicate.values?.[0]; - if (typeof v !== "string") { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - path: [...path, "values"], - message: `filter operator "${op}" requires a string value (got ${typeof v})`, - }); - } - } -} - -/** - * Iterative pre-parse depth check. Zod's union/object parsers walk the input - * recursively before our `superRefine` depth cap fires, so a deeply-nested - * `{ and: [{ and: [...] }] }` payload could stack-overflow during parse, never - * reaching the validator's depth check. This walk is iterative (explicit - * stack) and aborts as soon as `METRIC_FILTER_MAX_DEPTH` is exceeded, so a - * hostile payload of any size cannot drive the call stack. - * - * Walks BOTH `and` and `or` branches when both are present on the same node — - * Zod's `.strict()` rejects the multi-key shape downstream, but the pre-check - * has to inspect every branch Zod might recurse into (an earlier version used - * `else if` and was bypassed by `{ and: [], or: }`). - * - * Group-children breadth is also capped: a flat `{ and: [...10M items...] }` - * cannot push 10M frames onto the explicit stack here. Predicate leaves do NOT - * count toward depth — only nested `and` / `or` wrappers — matching the rule - * {@link validateFilterTree} enforces. - */ -function preCheckFilterDepth(filter: unknown): void { - if (filter == null || typeof filter !== "object") return; - const stack: Array<[unknown, number]> = [[filter, 0]]; - while (stack.length > 0) { - const popped = stack.pop(); - if (popped === undefined) continue; - const [node, depth] = popped; - if (node == null || typeof node !== "object") continue; - const obj = node as Record; - for (const groupKey of ["and", "or"] as const) { - const children = obj[groupKey]; - if (!Array.isArray(children)) continue; - if (children.length > METRIC_FILTER_GROUP_MAX) { - throw new ValidationError( - "Invalid metric request body (fields: filter)", - { - context: { - reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, - }, - }, - ); - } - if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { - throw new ValidationError( - "Invalid metric request body (fields: filter)", - { - context: { - reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, - }, - }, - ); - } - for (const child of children) { - stack.push([child, depth + 1]); - } - } - } -} - -/** - * Validate a `POST /api/analytics/metric/:key` request body against the static - * structured shape. Throws {@link ValidationError} (a 400 on the canonical - * error path) with the offending field paths; the raw values stay in telemetry - * context, never the public body. - * - * The recursion depth is bounded BEFORE Zod sees the input — the schema's own - * depth check fires inside `superRefine`, which only runs after Zod's recursive - * parse has already walked the tree on the call stack. - */ -export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { - if (body != null && typeof body === "object") { - preCheckFilterDepth((body as { filter?: unknown }).filter); - } - const result = metricRequestSchema.safeParse(body); - if (!result.success) { - const fieldPaths = result.error.issues - .map((i) => i.path.join(".") || "(root)") - .join(", "); - throw new ValidationError( - fieldPaths.length > 0 - ? `Invalid metric request body (fields: ${fieldPaths})` - : "Invalid metric request body", - { context: { issues: result.error.issues } }, - ); - } - return result.data; -} - -/** - * Construct the metric SQL. - * - * Shape: - * - * SELECT MEASURE(m) AS m[, …][, …] - * FROM - * [WHERE ] - * [GROUP BY ALL] - * [LIMIT n] - * - * Notes: - * - Every measure and dimension is validated by {@link isValidColumnName} and - * backtick-quoted by {@link quoteIdentifier} before it is interpolated - * (column references cannot be parameterized — they are SQL identifiers), - * and the FQN is validated and quoted by {@link quoteSafeFqn}. There is - * deliberately NO name allowlist — quoting is the security boundary. No - * user-supplied string reaches the SQL - * string without passing a grammar gate. - * - `GROUP BY ALL` is added when at least one dimension is requested. UC - * requires GROUP BY when MEASURE() is mixed with non-aggregated columns; - * `GROUP BY ALL` is the documented form that works without re-listing each - * dimension. - * - The `WHERE` clause is rendered from the recursive filter tree. Every value - * flows through Statement Execution's named bind-var path (`:f_`); no - * value is ever interpolated as a literal. - * - When `timeGrain` is set, the `timeDimension` column renders as - * `date_trunc('', ) AS ` (the grain is a grammar-gated - * single-quoted literal, not a bind param); other dimensions render bare — - * see {@link renderDimensionClause}. - * - * Returns `{ statement, parameters }` where `parameters` is the named bind-var - * dictionary the plugin's `query()` consumes. - */ -export function buildMetricSql( - registration: MetricRegistration, - request: IAnalyticsMetricRequest, -): { - statement: string; - parameters: Record; -} { - const quotedSource = quoteSafeFqn(registration.source); - - if (request.measures.length === 0) { - throw new Error("buildMetricSql requires at least one measure."); - } - - // Defense-in-depth re-gate: `validateMetricRequest` already rejects any name - // `isValidColumnName` refuses, but `buildMetricSql` is exported and may be - // reached without it. `quoteIdentifier` throws on a control/newline name, so - // the quoting below is itself the boundary; the explicit check keeps the - // error message uniform with the other interpolated tokens. - for (const m of request.measures) { - if (!isValidColumnName(m)) { - throw new Error( - `Refusing to build SQL: measure "${m}" is not a valid identifier.`, - ); - } - } - - const dimensions = request.dimensions ?? []; - for (const d of dimensions) { - if (!isValidColumnName(d)) { - throw new Error( - `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, - ); - } - } - - // Deterministic order so cache keys collapse semantically equivalent calls. - // Alias each measure to its plain name (backtick-quoted) so result rows have - // keys matching the registered measure (`{ "net-revenue": 1234 }`) rather - // than the SQL-function serialization Databricks returns by default. The - // warehouse reports the aliased column under the UNQUOTED name (backticks are - // delimiters, not part of the name), so `MEASURE(`x`) AS `x`` yields a row - // key of exactly `x` — preserved through result materialization. - const measureClauses = [...request.measures] - .sort() - .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); - - const dimensionClauses = [...dimensions] - .sort() - .map((d) => - renderDimensionClause(d, request.timeGrain, request.timeDimension), - ); - - const selectList = [...measureClauses, ...dimensionClauses].join(", "); - const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; - - const limitClause = - typeof request.limit === "number" && request.limit > 0 - ? ` LIMIT ${Math.floor(request.limit)}` - : ""; - - // Filter translation. Every value is bound through `:f_` named params; - // every column identifier is grammar-gated in `renderFilter`. Empty filter or - // no filter → no WHERE. - const parameters: Record = {}; - let whereClause = ""; - if (request.filter !== undefined) { - const fragment = renderFilter(request.filter, parameters, { - counter: 0, - depth: 0, - }); - if (fragment !== null && fragment.length > 0) { - whereClause = ` WHERE ${fragment}`; - } - } - - const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; - return { statement, parameters }; -} - -/** - * Mutable counter / depth threaded through {@link renderFilter}. Fresh per - * `buildMetricSql` call, so two requests never share bind-var indexes. - */ -interface FilterRenderState { - counter: number; - depth: number; -} - -/** - * Recursively render a filter tree into a SQL fragment, pushing bind values - * into `params` keyed by `:f_` names. - * - * Returns `null` for an empty group (no WHERE clause needed). The caller's - * `buildMetricSql` only emits `WHERE` when this returns a non-null, non-empty - * fragment. Empty `and: []` collapses to null (SQL's vacuous-truth semantics - * for AND); empty `or: []` renders `1 = 0` — the validator rejects `or: []` - * before the SQL builder, but if it slips through we render vacuous-false - * rather than dropping the predicate silently (defense in depth so a future - * validator bypass cannot turn `or: []` into "match everything"). - * - * Defense-in-depth: even though the request body's filter has already been - * validated, every member name is re-checked against {@link isValidColumnName} - * here and backtick-quoted. If validation is ever bypassed, the SQL - * constructor still refuses to interpolate a name it cannot safely quote. - */ -function renderFilter( - node: MetricFilter, - params: Record, - state: FilterRenderState, -): string | null { - if (node === null || typeof node !== "object") { - throw new Error( - "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", - ); - } - - if ("and" in node || "or" in node) { - const groupKey = "and" in node ? "and" : "or"; - if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { - throw new Error( - `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, - ); - } - - const children = ( - node as { and?: ReadonlyArray } & { - or?: ReadonlyArray; - } - )[groupKey]; - - if (!Array.isArray(children) || children.length === 0) { - if (groupKey === "or") { - return "1 = 0"; - } - return null; - } - - // Sort-before-hash discipline: within a group, predicate leaves are - // stable-sorted by (member, operator) before contributing to the rendered - // fragment, so semantically equivalent calls produce the same SQL string - // and (downstream) the same cache key. - const sortedChildren = sortFilterChildren(children); - - const fragments: string[] = []; - const childState: FilterRenderState = { - counter: state.counter, - depth: state.depth + 1, - }; - for (const child of sortedChildren) { - const rendered = renderFilter(child, params, childState); - if (rendered != null && rendered.length > 0) { - fragments.push(rendered); - } - } - state.counter = childState.counter; - - if (fragments.length === 0) return null; - if (fragments.length === 1) return fragments[0]; - const joiner = groupKey === "and" ? " AND " : " OR "; - return `(${fragments.join(joiner)})`; - } - - // Leaf predicate — grammar-gate the member and operator, then render. - const predicate = node as MetricPredicate; - - if (!isValidColumnName(predicate.member)) { - throw new Error( - `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, - ); - } - if ( - !METRIC_FILTER_OPERATORS.includes( - predicate.operator as MetricFilterOperatorName, - ) - ) { - throw new Error( - `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, - ); - } - - return renderPredicate(predicate, params, state); -} - -/** - * Stable-sort filter children inside an AND/OR group by `(member, operator)`. - * - * Predicates carry both fields and sort by their pair; nested groups sort - * after predicates and stay in their original relative order (a nested group - * is opaque from the outside — we cannot collapse it to a single key). This is - * the sort-before-hash invariant applied at the SQL-fragment level so - * downstream cache keys collapse semantically equivalent calls. - */ -function sortFilterChildren( - children: ReadonlyArray, -): MetricFilter[] { - const indexed = children.map((child, idx) => { - let key: string; - let isPredicate: boolean; - if ( - child !== null && - typeof child === "object" && - !("and" in child) && - !("or" in child) - ) { - const p = child as MetricPredicate; - // JSON.stringify the (member, operator) pair so the sort key is - // injective regardless of content. A plain delimiter is unsafe now that - // members accept the full delimited-identifier grammar (a member may - // contain "/", ".", etc.): `member "a/b" + op "c"` and - // `member "a" + op "b/c"` would both collapse to "a/b/c" under any - // single-char separator, tie-breaking distinct pairs to input order. - // JSON encoding escapes any separator, so distinct pairs map to distinct - // keys — the same technique canonicalizeFilter uses for values. - key = JSON.stringify([p.member, p.operator]); - isPredicate = true; - } else { - // Nested groups don't have a single (member, operator) — keep their - // original index so multiple nested groups within the same parent remain - // stable relative to each other. - key = ""; - isPredicate = false; - } - return { child, idx, key, isPredicate }; - }); - - indexed.sort((a, b) => { - if (a.isPredicate && !b.isPredicate) return -1; - if (!a.isPredicate && b.isPredicate) return 1; - if (a.isPredicate && b.isPredicate) { - if (a.key < b.key) return -1; - if (a.key > b.key) return 1; - } - return a.idx - b.idx; - }); - - return indexed.map((entry) => entry.child); -} - -/** - * Translate a single predicate into a SQL fragment. - * - * Every value flows through a freshly-allocated `:f_` named bind var. - * Nothing from the request body is ever interpolated as a literal — the - * fragment carries identifiers (grammar-gated) and operators (whitelisted), - * then references the bind name for each value. - * - * `set` and `notSet` emit `IS NULL` / `IS NOT NULL` with no bind value. `in` - * and `notIn` emit `IN (:f_0, :f_1, ...)`. `contains` and `notContains` emit - * `LIKE :f_0` / `NOT LIKE :f_0` and pre-bind the value with `%` wrapping. - */ -function renderPredicate( - predicate: MetricPredicate, - params: Record, - state: FilterRenderState, -): string { - // Backtick-quote the column identifier (defense-in-depth: the member was - // already validated by `isValidColumnName` in `renderFilter`). Quoting is - // what lets a delimited column name — hyphens, dots, non-ASCII — reach SQL - // safely; the bound value still flows through `:f_` params, never here. - const col = quoteIdentifier(predicate.member); - const op = predicate.operator; - const values = predicate.values ?? []; - - switch (op) { - case "equals": - return `${col} = ${bindValue(values[0], params, state)}`; - case "notEquals": - return `${col} <> ${bindValue(values[0], params, state)}`; - case "gt": - return `${col} > ${bindValue(values[0], params, state)}`; - case "gte": - return `${col} >= ${bindValue(values[0], params, state)}`; - case "lt": - return `${col} < ${bindValue(values[0], params, state)}`; - case "lte": - return `${col} <= ${bindValue(values[0], params, state)}`; - case "in": { - const placeholders = values.map((v) => bindValue(v, params, state)); - return `${col} IN (${placeholders.join(", ")})`; - } - case "notIn": { - const placeholders = values.map((v) => bindValue(v, params, state)); - return `${col} NOT IN (${placeholders.join(", ")})`; - } - case "contains": { - const raw = values[0]; - if (typeof raw !== "string") { - throw new Error( - `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, - ); - } - return `${col} LIKE ${bindLikeValue(raw, params, state)}`; - } - case "notContains": { - const raw = values[0]; - if (typeof raw !== "string") { - throw new Error( - `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, - ); - } - return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; - } - case "set": - return `${col} IS NOT NULL`; - case "notSet": - return `${col} IS NULL`; - default: { - // Exhaustiveness — the operator union is closed; if this is reached the - // operator vocabulary widened without updating the switch. - const _exhaustive: never = op; - throw new Error( - `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, - ); - } - } -} - -/** - * Allocate a fresh `:f_` bind name for `value`, push the typed marker into - * `params`, and return the placeholder string. Bumps the counter. - */ -function bindValue( - value: string | number | undefined, - params: Record, - state: FilterRenderState, -): string { - if (value === undefined) { - throw new Error( - "Refusing to build SQL: filter predicate is missing a required value.", - ); - } - const name = `f_${state.counter}`; - state.counter += 1; - if (typeof value === "number") { - params[name] = sqlHelpers.number(value); - } else if (typeof value === "string") { - params[name] = sqlHelpers.string(value); - } else { - throw new Error( - `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, - ); - } - return `:${name}`; -} - -/** - * Like {@link bindValue}, but wraps the value in `%...%` for `LIKE` / - * `NOT LIKE`. SQL wildcards in the user-supplied string remain in the value - * (matching the documented "contains" semantics) — escape-on-receive could be - * added later as an opt-in if customers request strict-substring matching. - */ -function bindLikeValue( - value: string, - params: Record, - state: FilterRenderState, -): string { - const name = `f_${state.counter}`; - state.counter += 1; - params[name] = sqlHelpers.string(`%${value}%`); - return `:${name}`; -} - -/** - * Render a single SELECT-list clause for a dimension. - * - * When `timeGrain` is set and `dim` is the request's `timeDimension`, the - * column is bucketed: `date_trunc('', ) AS `. Every other - * dimension renders bare. Both the grain literal and the column identifier are - * grammar-gated before they reach the SQL string: - * - * - The grain is single-quoted (it is a `date_trunc` unit literal, NOT a bind - * param), so it is re-checked against {@link TIME_GRAIN_PATTERN} here before - * interpolation. The schema also gates it, but `buildMetricSql` is exported - * and may be reached on a path that bypasses `validateMetricRequest`, so the - * grammar gate is applied at the interpolation point too — matching how - * every other interpolated identifier (measures, dimensions, `timeDimension`, - * the FQN) is re-gated in the builder. - * - The column cannot be parameterized (it is an identifier), so it is - * re-checked against {@link isValidColumnName} and backtick-quoted here as - * belt-and-suspenders even though the schema already gated `timeDimension`. - * - * The aliasing keeps result-row keys stable (`{ order_date: ... }`) regardless - * of whether the grain was applied. - */ -function renderDimensionClause( - dim: string, - timeGrain: string | undefined, - timeDimension: string | undefined, -): string { - if (timeGrain != null && dim === timeDimension) { - if (!isValidColumnName(dim)) { - throw new Error( - `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, - ); - } - // The grain is interpolated as a single-quoted `date_trunc` unit literal - // (NOT a bind param), so it stays gated by the narrow TIME_GRAIN_PATTERN — - // it is a keyword, not a delimited identifier. - if (!TIME_GRAIN_PATTERN.test(timeGrain)) { - throw new Error( - `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, - ); - } - const quoted = quoteIdentifier(dim); - return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; - } - return quoteIdentifier(dim); -} - -/** - * Compose the cache key for a metric-view request. - * - * Reserved namespace `metric` separates metric-view caches from query caches. - * The returned array is consumed by `CacheManager.generateKey`, which - * concatenates and sha256-hashes the parts — the per-concern structure keeps - * the key inspectable in tests and debug logs without giving up determinism. - * - * Sort-before-hash collapses semantically equivalent calls onto one entry: - * - `measures` / `dimensions`: lexicographic sort. - * - `filter`: predicates inside each AND/OR group are stable-sorted by - * `(member, operator)` and the group kind (`and` vs `or`) is preserved by - * {@link canonicalizeFilter}; values are included verbatim so distinct - * filter values yield distinct keys. - * - * `source` (the metric view's UC FQN) salts the key so that repointing a - * metric `key` to a different FQN in `metric-views.json` cannot serve rows - * cached under the old source within the TTL — `metricKey` alone is not enough - * because the key→source binding is mutable config. - * - * `timeGrain` salts the key whenever set. `timeDimension` only salts the key - * when `timeGrain` is also set, because `renderDimensionClause` only applies - * `date_trunc('', )` when a grain is present — with no - * grain the field has no effect on the emitted SQL, so including it would fork - * the cache on an input that produced identical SQL. - * - * `executorKey` is `"sp"` for SP-lane entries (shared cache) or a sha256 hash - * of the end user's identity for OBO-lane entries (per-user isolation) — see - * {@link deriveMetricExecutorKey}. - */ -export function composeMetricCacheKey(input: { - metricKey: string; - source: string; - measures: string[]; - dimensions?: string[]; - timeGrain?: string; - timeDimension?: string; - filter?: MetricFilter; - format: string; - executorKey: string; - limit?: number; -}): string[] { - const sortedMeasures = [...input.measures].sort(); - const sortedDimensions = [...(input.dimensions ?? [])].sort(); - const filterFingerprint = - input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; - // `timeDimension` only changes the SQL when `timeGrain` is set (see - // renderDimensionClause); keying on it otherwise would fork the cache on a - // no-op field. - const timeDimensionPart = - input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; - return [ - "metric", - input.metricKey, - input.source, - input.format, - sortedMeasures.join(","), - sortedDimensions.join(","), - input.timeGrain ?? "_", - timeDimensionPart, - filterFingerprint, - typeof input.limit === "number" ? String(input.limit) : "_", - input.executorKey, - ]; -} - -/** - * Derive the cache executor key from a metric registration's lane and the - * caller's user identity. - * - * Returns `"sp"` for SP-lane entries (every caller shares the cache) and a - * sha256 hex digest of the user identity for OBO-lane entries (each user gets - * an isolated cache scope). - * - * The user identity is hashed — never stored verbatim — so the cache layer - * (which logs keys at debug level and persists them in any cache backend) - * never sees raw user emails or principal names. A stable, opaque token is - * exactly what we need: same user → same key (so cache hits work), different - * users → different keys (so isolation holds), and reverse lookup is - * infeasible. - * - * For OBO requests without a resolvable identity (missing or whitespace-only - * user id), throw `AuthenticationError.missingUserId()` rather than falling - * back to a shared `"anonymous"` sentinel — distinct misconfigured callers - * would otherwise collide into one cache scope and read each other's cached - * results. The route computes this inside its try/catch, so the throw lands on - * the canonical 401 envelope. - */ -export function deriveMetricExecutorKey(input: { - lane: MetricLane; - userIdentity?: string | null; -}): string { - if (input.lane === "sp") { - return "sp"; - } - // OBO lane — hash the user identity so the raw email/principal never reaches - // the cache layer. Missing/whitespace identity is a hard auth failure: the - // alternative ("anonymous" sentinel) collides every misconfigured caller - // into a single cache scope, so user A's results could leak to user B. - const identity = input.userIdentity?.trim(); - if (!identity) { - throw AuthenticationError.missingUserId(); - } - return createHash("sha256").update(identity).digest("hex"); -} - -/** - * Produce a deterministic string fingerprint of the filter tree. - * - * The fingerprint sorts predicates within each AND/OR group by - * `(member, operator)` and recursively canonicalizes nested groups. Values are - * included verbatim so cache entries differ when the filter targets different - * values (`region in [EMEA]` vs `region in [APAC]` → different keys; `equals A` - * vs `equals B` → different keys), while order-insensitive predicate lists - * collapse to the same key. - */ -function canonicalizeFilter(node: MetricFilter): string { - if (node === null || typeof node !== "object") { - return "_"; - } - - if ("and" in node || "or" in node) { - const groupKey = "and" in node ? "and" : "or"; - const children = ( - node as { and?: ReadonlyArray } & { - or?: ReadonlyArray; - } - )[groupKey]; - - if (!Array.isArray(children) || children.length === 0) { - return `${groupKey}()`; - } - - const sorted = sortFilterChildren(children); - const childFingerprints = sorted.map(canonicalizeFilter); - return `${groupKey}(${childFingerprints.join(",")})`; - } - - // Leaf predicate. JSON.stringify every structural part — member, operator, - // and each value (with its type) — so no content can be confused with a - // separator. This matters now that `member` accepts the full delimited- - // identifier grammar (it may contain "/", ".", etc.): a bare - // `p(${member}/${operator}/...)` would let `member "a", op "b"` collide with - // `member "a/b"` and fork/merge cache entries. Encoding the whole tuple as - // JSON makes the fingerprint injective regardless of member/value content. - const p = node as MetricPredicate; - const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); - return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; -} +export * from "./mv"; diff --git a/packages/appkit/src/plugins/analytics/mv/cache.ts b/packages/appkit/src/plugins/analytics/mv/cache.ts new file mode 100644 index 00000000..45de6377 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/cache.ts @@ -0,0 +1,71 @@ +import { createHash } from "node:crypto"; +import { AuthenticationError } from "../../../errors"; +import type { MetricFilter, MetricLane, MetricPredicate } from "../types"; +import { sortFilterChildren } from "./formatters"; +import type { MetricCacheKeyInput } from "./types"; + +export function composeMetricCacheKey(input: MetricCacheKeyInput): string[] { + const sortedMeasures = [...input.measures].sort(); + const sortedDimensions = [...(input.dimensions ?? [])].sort(); + const filterFingerprint = + input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + // `timeDimension` only changes the SQL when `timeGrain` is set (see + // renderDimensionClause); keying on it otherwise would fork the cache on a + // no-op field. + const timeDimensionPart = + input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; + return [ + "metric", + input.metricKey, + input.source, + input.format, + sortedMeasures.join(","), + sortedDimensions.join(","), + input.timeGrain ?? "_", + timeDimensionPart, + filterFingerprint, + typeof input.limit === "number" ? String(input.limit) : "_", + input.executorKey, + ]; +} + +export function deriveMetricExecutorKey(input: { + lane: MetricLane; + userIdentity?: string | null; +}): string { + if (input.lane === "sp") { + return "sp"; + } + const identity = input.userIdentity?.trim(); + if (!identity) { + throw AuthenticationError.missingUserId(); + } + return createHash("sha256").update(identity).digest("hex"); +} + +function canonicalizeFilter(node: MetricFilter): string { + if (node === null || typeof node !== "object") { + return "_"; + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + return `${groupKey}()`; + } + + const sorted = sortFilterChildren(children); + const childFingerprints = sorted.map(canonicalizeFilter); + return `${groupKey}(${childFingerprints.join(",")})`; + } + + const p = node as MetricPredicate; + const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); + return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; +} diff --git a/packages/appkit/src/plugins/analytics/mv/constants.ts b/packages/appkit/src/plugins/analytics/mv/constants.ts new file mode 100644 index 00000000..3676c80b --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/constants.ts @@ -0,0 +1,122 @@ +import path from "node:path"; +import type { MetricFilterOperatorName, MetricLane } from "../types"; + +/** + * Default queries directory. Mirrors `AppManager`'s + * `path.resolve(process.cwd(), "config/queries")` so dev mode and production + * share a single source of truth for where metric config lives. Exported so + * `AnalyticsPlugin` can default `config.queriesDir` to the same path. + */ +export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +export const METRIC_CONFIG_FILE = "metric-views.json"; + +/** + * Measure, dimension, and filter-member names are **column identifiers**: they + * are validated by the shared {@link isValidColumnName} (rejects only control + * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at + * every interpolation point. Quoting — not a narrow ASCII allowlist — is the + * injection boundary, so the runtime accepts the full delimited-identifier + * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). + * There is deliberately NO name allowlist: a well-formed-but-unknown column + * falls through to the warehouse and surfaces as a sanitized canonical error. + * + * Time-grain token shape. Unlike the column identifiers above, the grain is + * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, + * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a + * narrow keyword-shaped gate — that pattern is what keeps a hostile token out + * of the quoted-literal position. + */ +export const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; + +/** + * The exact twelve filter operators allowed at v1. The runtime tuple is the + * server-side source of truth; the client-side type union + * `MetricFilterOperatorName` mirrors these names statically. + */ +export const METRIC_FILTER_OPERATORS = [ + "equals", + "notEquals", + "in", + "notIn", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", + "set", + "notSet", +] as const satisfies readonly MetricFilterOperatorName[]; + +/** + * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — + * enough for any real BI filter UI, low enough that a hostile or malformed + * payload cannot stack-overflow the recursive validator or translator. + * + * The depth count is the number of nested `{ and }` / `{ or }` wrappers + * encountered while descending — leaf predicates do not count toward depth. + */ +export const METRIC_FILTER_MAX_DEPTH = 8; + +/** + * Cardinality caps on user-controlled arrays. Closes the recurring + * `unbounded-request-parameters` finding: a hostile caller could otherwise + * send `values: [...10M items...]` and exhaust the validator + the named + * bind-var binding step. The limits below are deliberately generous — higher + * than any real BI UI would emit — so legitimate traffic never trips them. + */ +export const METRIC_MEASURES_MAX = 50; +export const METRIC_DIMENSIONS_MAX = 20; +export const METRIC_FILTER_VALUES_MAX = 1000; +export const METRIC_LIMIT_MAX = 100_000; + +/** + * Maximum number of children per AND/OR group node. Without this cap a single + * flat group like `{ and: [...10M empty objects...] }` would push tens of + * millions of frames onto the iterative pre-check's stack — OOM before + * validation even gets to Zod. The Zod schema enforces the same cap so the + * rejection point is consistent regardless of which validator catches it + * first. + */ +export const METRIC_FILTER_GROUP_MAX = 100; + +/** Operators that require exactly one value. */ +export const SINGLE_VALUE_OPERATORS = new Set([ + "equals", + "notEquals", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", +]); + +/** Operators that require at least one value. */ +export const LIST_VALUE_OPERATORS = new Set([ + "in", + "notIn", +]); + +/** Operators that reject `values` entirely. */ +export const NULL_OPERATORS = new Set([ + "set", + "notSet", +]); + +/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ +export const STRING_OPERATORS = new Set([ + "contains", + "notContains", +]); + +/** + * Map an entry's declared `executor` to the internal execution lane: + * - `"user"` → `"obo"` (per-user cache, on-behalf-of) + * - `"app_service_principal"` (default) → `"sp"` (shared cache) + */ +export function laneFromExecutor( + executor: "app_service_principal" | "user", +): MetricLane { + return executor === "user" ? "obo" : "sp"; +} diff --git a/packages/appkit/src/plugins/analytics/mv/formatters.ts b/packages/appkit/src/plugins/analytics/mv/formatters.ts new file mode 100644 index 00000000..57ac74de --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/formatters.ts @@ -0,0 +1,319 @@ +import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; +import { + isValidColumnName, + isValidFqn, + quoteFqnForSql, + quoteIdentifier, +} from "../../../../../shared/src/schemas/metric-fqn"; +import type { + IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, + MetricPredicate, + MetricRegistration, +} from "../types"; +import { + METRIC_FILTER_MAX_DEPTH, + METRIC_FILTER_OPERATORS, + TIME_GRAIN_PATTERN, +} from "./constants"; +import type { FilterRenderState } from "./types"; + +function quoteSafeFqn(fqn: string): string { + if (!isValidFqn(fqn)) { + throw new Error( + `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, + ); + } + return quoteFqnForSql(fqn); +} + +export function buildMetricSql( + registration: MetricRegistration, + request: IAnalyticsMetricRequest, +): { + statement: string; + parameters: Record; +} { + const quotedSource = quoteSafeFqn(registration.source); + + if (request.measures.length === 0) { + throw new Error("buildMetricSql requires at least one measure."); + } + + for (const m of request.measures) { + if (!isValidColumnName(m)) { + throw new Error( + `Refusing to build SQL: measure "${m}" is not a valid identifier.`, + ); + } + } + + const dimensions = request.dimensions ?? []; + for (const d of dimensions) { + if (!isValidColumnName(d)) { + throw new Error( + `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, + ); + } + } + + const measureClauses = [...request.measures] + .sort() + .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); + + const dimensionClauses = [...dimensions] + .sort() + .map((d) => + renderDimensionClause(d, request.timeGrain, request.timeDimension), + ); + + const selectList = [...measureClauses, ...dimensionClauses].join(", "); + const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; + + const limitClause = + typeof request.limit === "number" && request.limit > 0 + ? ` LIMIT ${Math.floor(request.limit)}` + : ""; + + const parameters: Record = {}; + let whereClause = ""; + if (request.filter !== undefined) { + const fragment = renderFilter(request.filter, parameters, { + counter: 0, + depth: 0, + }); + if (fragment !== null && fragment.length > 0) { + whereClause = ` WHERE ${fragment}`; + } + } + + const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; + return { statement, parameters }; +} + +function renderFilter( + node: MetricFilter, + params: Record, + state: FilterRenderState, +): string | null { + if (node === null || typeof node !== "object") { + throw new Error( + "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", + ); + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new Error( + `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, + ); + } + + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + if (groupKey === "or") { + return "1 = 0"; + } + return null; + } + + const sortedChildren = sortFilterChildren(children); + + const fragments: string[] = []; + const childState: FilterRenderState = { + counter: state.counter, + depth: state.depth + 1, + }; + for (const child of sortedChildren) { + const rendered = renderFilter(child, params, childState); + if (rendered != null && rendered.length > 0) { + fragments.push(rendered); + } + } + state.counter = childState.counter; + + if (fragments.length === 0) return null; + if (fragments.length === 1) return fragments[0]; + const joiner = groupKey === "and" ? " AND " : " OR "; + return `(${fragments.join(joiner)})`; + } + + const predicate = node as MetricPredicate; + + if (!isValidColumnName(predicate.member)) { + throw new Error( + `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, + ); + } + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + throw new Error( + `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, + ); + } + + return renderPredicate(predicate, params, state); +} + +export function sortFilterChildren( + children: ReadonlyArray, +): MetricFilter[] { + const indexed = children.map((child, idx) => { + let key: string; + let isPredicate: boolean; + if ( + child !== null && + typeof child === "object" && + !("and" in child) && + !("or" in child) + ) { + const p = child as MetricPredicate; + key = JSON.stringify([p.member, p.operator]); + isPredicate = true; + } else { + key = ""; + isPredicate = false; + } + return { child, idx, key, isPredicate }; + }); + + indexed.sort((a, b) => { + if (a.isPredicate && !b.isPredicate) return -1; + if (!a.isPredicate && b.isPredicate) return 1; + if (a.isPredicate && b.isPredicate) { + if (a.key < b.key) return -1; + if (a.key > b.key) return 1; + } + return a.idx - b.idx; + }); + + return indexed.map((entry) => entry.child); +} + +function renderPredicate( + predicate: MetricPredicate, + params: Record, + state: FilterRenderState, +): string { + const col = quoteIdentifier(predicate.member); + const op = predicate.operator; + const values = predicate.values ?? []; + + switch (op) { + case "equals": + return `${col} = ${bindValue(values[0], params, state)}`; + case "notEquals": + return `${col} <> ${bindValue(values[0], params, state)}`; + case "gt": + return `${col} > ${bindValue(values[0], params, state)}`; + case "gte": + return `${col} >= ${bindValue(values[0], params, state)}`; + case "lt": + return `${col} < ${bindValue(values[0], params, state)}`; + case "lte": + return `${col} <= ${bindValue(values[0], params, state)}`; + case "in": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} IN (${placeholders.join(", ")})`; + } + case "notIn": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} NOT IN (${placeholders.join(", ")})`; + } + case "contains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} LIKE ${bindLikeValue(raw, params, state)}`; + } + case "notContains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; + } + case "set": + return `${col} IS NOT NULL`; + case "notSet": + return `${col} IS NULL`; + default: { + const _exhaustive: never = op; + throw new Error( + `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, + ); + } + } +} + +function bindValue( + value: string | number | undefined, + params: Record, + state: FilterRenderState, +): string { + if (value === undefined) { + throw new Error( + "Refusing to build SQL: filter predicate is missing a required value.", + ); + } + const name = `f_${state.counter}`; + state.counter += 1; + if (typeof value === "number") { + params[name] = sqlHelpers.number(value); + } else if (typeof value === "string") { + params[name] = sqlHelpers.string(value); + } else { + throw new Error( + `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, + ); + } + return `:${name}`; +} + +function bindLikeValue( + value: string, + params: Record, + state: FilterRenderState, +): string { + const name = `f_${state.counter}`; + state.counter += 1; + params[name] = sqlHelpers.string(`%${value}%`); + return `:${name}`; +} + +function renderDimensionClause( + dim: string, + timeGrain: string | undefined, + timeDimension: string | undefined, +): string { + if (timeGrain != null && dim === timeDimension) { + if (!isValidColumnName(dim)) { + throw new Error( + `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, + ); + } + if (!TIME_GRAIN_PATTERN.test(timeGrain)) { + throw new Error( + `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, + ); + } + const quoted = quoteIdentifier(dim); + return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; + } + return quoteIdentifier(dim); +} diff --git a/packages/appkit/src/plugins/analytics/mv/index.ts b/packages/appkit/src/plugins/analytics/mv/index.ts new file mode 100644 index 00000000..de5eec8f --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/index.ts @@ -0,0 +1,9 @@ +export { composeMetricCacheKey, deriveMetricExecutorKey } from "./cache"; +export { QUERIES_DIR } from "./constants"; +export { buildMetricSql } from "./formatters"; +export { + __resetMetricRegistryCache, + getMetricRegistry, + loadMetricRegistry, +} from "./registry"; +export { validateMetricRequest } from "./schemas"; diff --git a/packages/appkit/src/plugins/analytics/mv/registry.ts b/packages/appkit/src/plugins/analytics/mv/registry.ts new file mode 100644 index 00000000..190fd1a6 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/registry.ts @@ -0,0 +1,192 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +// Canonical metric-source schema — the single source of truth for +// `metric-views.json`. Imported from the shared source directly (matching the +// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the +// same tree) so the runtime and the generated JSON schema validate identically. +import { metricSourceSchema } from "../../../../../shared/src/schemas/metric-source"; +import { createLogger } from "../../../logging/logger"; +import type { MetricRegistration } from "../types"; +import { laneFromExecutor, METRIC_CONFIG_FILE, QUERIES_DIR } from "./constants"; +import type { RegistryCacheSignature } from "./types"; + +const logger = createLogger("analytics:metric"); + +/** + * Read and validate `config/queries/metric-views.json` into a metric registry. + * + * Async and stateless — registration is a pure config parse with no warehouse + * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single + * `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps + * the event loop free: metric views are already heavier than a plain `.sql` + * query on the warehouse side, so the SDK layer must not add a blocking read + * on top. Caching + mtime-revalidation is layered on top by + * {@link getMetricRegistry} — this function always hits disk. + * + * Returns an empty registry when the file is absent: the metric-view path is + * additive and dormant until an app opts in by adding the config. A malformed + * file (unreadable, invalid JSON, or schema violation) throws — the caller + * surfaces a 503 rather than masking a broken deployment as a 404 for every + * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the + * file heals on the next request. + */ +export async function loadMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let raw: string; + try { + raw = await fs.readFile(metricPath, "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return Object.create(null); + } + throw err; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, + ); + } + + const result = metricSourceSchema.safeParse(parsed); + if (!result.success) { + const issues = result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; "); + throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); + } + + // Null-prototype map so a metric key that collides with an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) + // cannot resolve to a truthy non-registration at the `registry[key]` read + // site and slip past the unknown-key 404. Keys are still grammar-gated by + // `metricKeySchema` (identifier shape), but the null prototype removes the + // whole class of inherited-property lookups as a boundary. + const registry: Record = Object.create(null); + for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { + registry[key] = { + key, + source: entry.source, + lane: laneFromExecutor(entry.executor), + }; + } + + logger.debug( + "Loaded metric registry: %d entry(ies)", + Object.keys(registry).length, + ); + return registry; +} + +/** + * Module-level registry cache, keyed by the resolved queries directory. + * + * Keyed by DIR (not by plugin instance) because the registry is a pure + * function of the config file at that path — warehouse-independent, and two + * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see + * the same registry. Instance state would parse the identical file twice and + * risk divergence; a dir-keyed module cache shares one parse. + */ +const metricRegistryCache = new Map< + string, + { + signature: RegistryCacheSignature; + registry: Record; + } +>(); + +/** + * Clear the module-level registry cache. + * + * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the + * process; production never needs to clear it (a changed file is picked up via + * the stat signature). Tests that reuse a directory — or assert cold-load + * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional + * rather than relying on each test happening to use a unique temp dir. + */ +export function __resetMetricRegistryCache(): void { + metricRegistryCache.clear(); +} + +/** + * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only + * when `metric-views.json` has changed since the cached copy. + * + * Behaves like the sibling `.sql` query path (which re-reads per request) but + * cheaper: the steady-state cost is a single async `stat`, and the read + + * `JSON.parse` + zod validation are skipped when the file's + * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed + * semantics without the old permanent memo: + * + * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next + * request re-parses and serves the new registry with no server restart. + * - **Self-heal** — a load failure is NOT cached (we only populate the cache + * on a successful parse), so a fixed config is picked up on the next + * request instead of latching a 503 forever. + * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding + * the file later is picked up on the next request. + * + * Concurrency: two simultaneous cold requests may both `stat`+read before + * either populates the cache. That is a harmless redundant read of the same + * file (the sibling `.sql` path does not single-flight either), so no lock is + * taken. + * + * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so + * the route can surface a 503; the cache is left untouched on failure. + */ +export async function getMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let signature: RegistryCacheSignature | null; + try { + const stats = await fs.stat(metricPath); + signature = { + ctimeMs: stats.ctimeMs, + mtimeMs: stats.mtimeMs, + size: stats.size, + }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + // Absent file → dormant. Drop any stale cache entry (the file may have + // been deleted) and return an empty registry without touching the cache. + metricRegistryCache.delete(queriesDir); + signature = null; + } else { + // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal + // for THIS request → the route surfaces a 503, consistent with the + // malformed-config → 503 path. It is not latched: with the self-heal + // design a transient error clears on the next request, which is strictly + // better than the old memo that could latch-and-serve-stale. + throw err; + } + } + + if (signature === null) { + return Object.create(null); + } + + const cached = metricRegistryCache.get(queriesDir); + if ( + cached !== undefined && + cached.signature.ctimeMs === signature.ctimeMs && + cached.signature.mtimeMs === signature.mtimeMs && + cached.signature.size === signature.size + ) { + return cached.registry; + } + + // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed + // file never latches — the next request re-attempts and heals. + const registry = await loadMetricRegistry(queriesDir); + metricRegistryCache.set(queriesDir, { signature, registry }); + return registry; +} diff --git a/packages/appkit/src/plugins/analytics/mv/schemas.ts b/packages/appkit/src/plugins/analytics/mv/schemas.ts new file mode 100644 index 00000000..0f00dcad --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/schemas.ts @@ -0,0 +1,356 @@ +import { z } from "zod"; +import { isValidColumnName } from "../../../../../shared/src/schemas/metric-fqn"; +import { ValidationError } from "../../../errors"; +import type { + IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, + MetricPredicate, +} from "../types"; +import { normalizeAnalyticsFormat } from "../types"; +import { + LIST_VALUE_OPERATORS, + METRIC_DIMENSIONS_MAX, + METRIC_FILTER_GROUP_MAX, + METRIC_FILTER_MAX_DEPTH, + METRIC_FILTER_OPERATORS, + METRIC_FILTER_VALUES_MAX, + METRIC_LIMIT_MAX, + METRIC_MEASURES_MAX, + NULL_OPERATORS, + SINGLE_VALUE_OPERATORS, + STRING_OPERATORS, + TIME_GRAIN_PATTERN, +} from "./constants"; + +/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ +const filterPredicateSchema: z.ZodType = z + .object({ + member: z + .string() + .min(1, { message: "filter predicate 'member' cannot be empty" }) + .refine(isValidColumnName, { + message: + "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + operator: z.string().min(1, { + message: "filter predicate 'operator' cannot be empty", + }) as z.ZodType, + values: z + .array(z.union([z.string(), z.number()])) + .max(METRIC_FILTER_VALUES_MAX, { + message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, + }) + .optional(), + }) + .strict(); + +/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ +const filterSchema: z.ZodType = z.lazy(() => + z.union([ + filterPredicateSchema, + z + .object({ + and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + z + .object({ + or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + ]), +); + +const metricRequestSchema = z + .object({ + measures: z + .array( + z + .string() + .min(1, "measure name cannot be empty") + .refine(isValidColumnName, { + message: + "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) + .min(1, "at least one measure is required") + .max(METRIC_MEASURES_MAX, { + message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, + }), + dimensions: z + .array( + z + .string() + .min(1, "dimension name cannot be empty") + .refine(isValidColumnName, { + message: + "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) + .max(METRIC_DIMENSIONS_MAX, { + message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, + }) + .optional(), + filter: filterSchema.optional(), + // Grammar-shaped bucketing grain, applied to `timeDimension` via + // `date_trunc`. The token is validated for safety here; the grain literal + // is interpolated (single-quoted, never a bind param) in + // `renderDimensionClause`, so this pattern gate is the security boundary. + timeGrain: z + .string() + .min(1, { message: "timeGrain cannot be empty" }) + .regex(TIME_GRAIN_PATTERN, { + message: "timeGrain must match /^[a-z][a-z_]*$/", + }) + .optional(), + // The single dimension `timeGrain` applies to via `date_trunc`. A column + // identifier (backtick-quoted at interpolation), so it accepts the full + // delimited-identifier grammar. Cross-field rules in `superRefine`: + // required when `timeGrain` is set, and must be one of `dimensions`. + timeDimension: z + .string() + .min(1, { message: "timeDimension cannot be empty" }) + .refine(isValidColumnName, { + message: + "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", + }) + .optional(), + limit: z + .number() + .int({ message: "limit must be an integer" }) + .positive({ message: "limit must be positive" }) + .max(METRIC_LIMIT_MAX, { + message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, + }) + .optional(), + format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.filter != null) { + validateFilterTree(value.filter, ctx, ["filter"], 0); + } + + const seen = new Set(); + const collided = new Set(); + for (const name of [...value.measures, ...(value.dimensions ?? [])]) { + if (seen.has(name)) { + collided.add(name); + } + seen.add(name); + } + if (collided.size > 0) { + ctx.addIssue({ + code: "custom", + message: + "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", + path: ["measures"], + }); + } + + if ( + value.format != null && + normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" + ) { + ctx.addIssue({ + code: "custom", + message: + "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", + path: ["format"], + }); + } + + if (value.timeGrain != null && value.timeDimension == null) { + ctx.addIssue({ + code: "custom", + message: "timeDimension is required when timeGrain is set", + path: ["timeDimension"], + }); + } + if ( + value.timeDimension != null && + !(value.dimensions ?? []).includes(value.timeDimension) + ) { + ctx.addIssue({ + code: "custom", + message: "timeDimension must be one of dimensions", + path: ["timeDimension"], + }); + } + }) as z.ZodType; + +function validateFilterTree( + node: MetricFilter, + ctx: z.RefinementCtx, + path: Array, + depth: number, +): void { + if (node === null || typeof node !== "object") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: "filter node must be a Predicate or { and } / { or } group", + }); + return; + } + + if ("and" in node || "or" in node) { + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }); + return; + } + + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, groupKey], + message: `filter ${groupKey} group must be an array of predicates or nested groups`, + }); + return; + } + + if (groupKey === "or" && children.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "or"], + message: "filter 'or' group must contain at least one predicate", + }); + return; + } + + children.forEach((child, idx) => { + validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); + }); + return; + } + + const predicate = node as MetricPredicate; + + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "operator"], + message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, + }); + return; + } + + const op = predicate.operator; + const values = predicate.values; + const valuesLen = values?.length ?? 0; + + if (NULL_OPERATORS.has(op)) { + if (values != null && valuesLen > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" must not carry values`, + }); + } + } else if (SINGLE_VALUE_OPERATORS.has(op)) { + if (valuesLen !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, + }); + } + } else if (LIST_VALUE_OPERATORS.has(op)) { + if (valuesLen < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires at least one value`, + }); + } + } + + if (STRING_OPERATORS.has(op) && valuesLen > 0) { + const v = predicate.values?.[0]; + if (typeof v !== "string") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires a string value (got ${typeof v})`, + }); + } + } +} + +function preCheckFilterDepth(filter: unknown): void { + if (filter == null || typeof filter !== "object") return; + const stack: Array<[unknown, number]> = [[filter, 0]]; + while (stack.length > 0) { + const popped = stack.pop(); + if (popped === undefined) continue; + const [node, depth] = popped; + if (node == null || typeof node !== "object") continue; + const obj = node as Record; + for (const groupKey of ["and", "or"] as const) { + const children = obj[groupKey]; + if (!Array.isArray(children)) continue; + if (children.length > METRIC_FILTER_GROUP_MAX) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, + }, + }, + ); + } + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }, + }, + ); + } + for (const child of children) { + stack.push([child, depth + 1]); + } + } + } +} + +export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + if (body != null && typeof body === "object") { + preCheckFilterDepth((body as { filter?: unknown }).filter); + } + const result = metricRequestSchema.safeParse(body); + if (!result.success) { + const fieldPaths = result.error.issues + .map((i) => i.path.join(".") || "(root)") + .join(", "); + throw new ValidationError( + fieldPaths.length > 0 + ? `Invalid metric request body (fields: ${fieldPaths})` + : "Invalid metric request body", + { context: { issues: result.error.issues } }, + ); + } + return result.data; +} diff --git a/packages/appkit/src/plugins/analytics/mv/types.ts b/packages/appkit/src/plugins/analytics/mv/types.ts new file mode 100644 index 00000000..be33cf7d --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/types.ts @@ -0,0 +1,25 @@ +import type { MetricFilter } from "../types"; + +export interface RegistryCacheSignature { + ctimeMs: number; + mtimeMs: number; + size: number; +} + +export interface FilterRenderState { + counter: number; + depth: number; +} + +export interface MetricCacheKeyInput { + metricKey: string; + source: string; + measures: string[]; + dimensions?: string[]; + timeGrain?: string; + timeDimension?: string; + filter?: MetricFilter; + format: string; + executorKey: string; + limit?: number; +}