diff --git a/apps/dev-playground/shared/appkit-types/metric-views.d.ts b/apps/dev-playground/shared/appkit-types/metric-views.d.ts index e0ada374..1c7fb87a 100644 --- a/apps/dev-playground/shared/appkit-types/metric-views.d.ts +++ b/apps/dev-playground/shared/appkit-types/metric-views.d.ts @@ -30,23 +30,31 @@ declare module "@databricks/appkit-ui/react" { measures: { "active_accounts": { type: "bigint"; + display_name: "Active Accounts"; + format: "#,##0"; }; "churn_rate": { type: "decimal"; + display_name: "Churn Rate"; }; "avg_ltv": { type: "double"; + display_name: "Average LTV"; + format: "$#,##0.00"; }; }; dimensions: { "segment": { type: "string"; + display_name: "Customer Segment"; }; "region": { type: "string"; + display_name: "Region"; }; "csm_email": { type: "string"; + display_name: "CSM Email"; }; }; }; @@ -80,27 +88,38 @@ declare module "@databricks/appkit-ui/react" { measures: { "mrr": { type: "double"; + display_name: "Monthly Recurring Revenue"; + format: "$#,##0.00"; }; "arr": { type: "double"; + display_name: "Annual Recurring Revenue"; + format: "$#,##0.00"; description: "Annualized contract value across all active subscriptions"; }; "new_arr": { type: "double"; + display_name: "New ARR"; + format: "$#,##0.00"; }; "churned_arr": { type: "double"; + display_name: "Churned ARR"; + format: "$#,##0.00"; }; }; dimensions: { "region": { type: "string"; + display_name: "Region"; }; "segment": { type: "string"; + display_name: "Customer Segment"; }; "created_at": { type: "timestamp_ltz"; + display_name: "Subscription Start"; time_grain: readonly ["day", "hour", "minute", "month", "quarter", "week", "year"]; }; }; diff --git a/docs/static/schemas/metric-source.schema.json b/docs/static/schemas/metric-source.schema.json index 1976ea7c..e606b92d 100644 --- a/docs/static/schemas/metric-source.schema.json +++ b/docs/static/schemas/metric-source.schema.json @@ -21,6 +21,7 @@ "properties": { "source": { "type": "string", + "maxLength": 767, "pattern": "^[^\\x00-\\x20\\x7f./]+\\.[^\\x00-\\x20\\x7f./]+\\.[^\\x00-\\x20\\x7f./]+$", "description": "Three-part Unity Catalog FQN of the metric view: ..", "examples": [ diff --git a/packages/appkit/src/plugins/analytics/analytics.ts b/packages/appkit/src/plugins/analytics/analytics.ts index 4ad05cc3..451373cc 100644 --- a/packages/appkit/src/plugins/analytics/analytics.ts +++ b/packages/appkit/src/plugins/analytics/analytics.ts @@ -29,6 +29,14 @@ import { Plugin, toPlugin } from "../../plugin"; import type { PluginManifest } from "../../registry"; import { queryDefaults } from "./defaults"; import manifest from "./manifest.json"; +import { + buildMetricSql, + composeMetricCacheKey, + QUERIES_DIR as DEFAULT_QUERIES_DIR, + deriveMetricExecutorKey, + getMetricRegistry, + validateMetricRequest, +} from "./metric"; import { QueryProcessor } from "./query"; import { type ArrowCapability, @@ -41,6 +49,7 @@ import { type AnalyticsStreamMessage, type IAnalyticsConfig, type IAnalyticsQueryRequest, + type MetricRegistration, normalizeAnalyticsFormat, type WarehouseStatus, } from "./types"; @@ -116,10 +125,19 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { */ private _arrowCapability = new Map(); + /** + * Directory the metric-view registry is read from (holds + * `metric-views.json`). Defaults to `config/queries/` under the process cwd + * — the same directory the `.sql` query path uses. Overridable ONLY via + * `config.queriesDir` for tests (see {@link IAnalyticsConfig.queriesDir}). + */ + private readonly _queriesDir: string; + constructor(config: IAnalyticsConfig) { super(config); this.config = config; this.queryProcessor = new QueryProcessor(); + this._queriesDir = config.queriesDir ?? DEFAULT_QUERIES_DIR; this.SQLClient = new SQLWarehouseConnector({ timeout: config.timeout, @@ -137,6 +155,19 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { }, }); + // Metric-view route. Registered parallel to `/query`; measures a + // registered UC Metric View over the same SSE envelope. Dormant until + // `config/queries/metric-views.json` exists (no config → empty registry → + // 404, nothing executes). + this.route(router, { + name: "metric", + method: "post", + path: "/metric/:key", + handler: async (req: express.Request, res: express.Response) => { + await this._handleMetricRoute(req, res); + }, + }); + // Column-names fallback for very wide Arrow schemas whose names don't fit // in the `X-Appkit-Arrow-Columns` response header (see // `_setArrowColumnsHeader`). The client hits this with the statement id @@ -426,6 +457,281 @@ export class AnalyticsPlugin extends Plugin implements ToolProvider { ); } + /** + * Handle metric-view execution requests (`POST /api/analytics/metric/:key`). + * + * Mirrors {@link _handleQueryRoute}'s JSON SSE path: the outer + * `executeStream` disables cache/retry and streams warehouse-readiness + * (`warehouse_status`) events, then the inner `execute` builds the metric SQL + * and delivers rows through {@link deliverJsonResult} as a `result` message. + * The `originalError` re-throw discipline preserves each error's structured + * `errorCode`/`clientMessage` for the SSE error payload. + * + * Lane dispatch is driven by the registration: an SP-lane metric runs as the + * app service principal (shared cache); an OBO-lane metric runs + * on-behalf-of the requesting user via `asUser(req)` (per-user cache keyed by + * a hash of the user identity). The executor + key are computed inside a + * try so a missing/whitespace OBO identity lands on the canonical 401 path. + */ + async _handleMetricRoute( + req: express.Request, + res: express.Response, + ): Promise { + const { key } = req.params; + + logger.debug(req, "Executing metric: %s", key); + + const event = logger.event(req); + event?.setComponent("analytics", "executeMetric").setContext("analytics", { + metric_key: key, + plugin: this.name, + }); + + if (!key) { + res.status(400).json({ error: "metric key is required" }); + return; + } + + // Resolve the registry from disk (cached + mtime-revalidated by + // `getMetricRegistry`, so the steady-state cost is one `stat`). A malformed + // config throws → 503, distinguishing a broken deployment from an unknown + // key (404). The failure is NOT latched: `getMetricRegistry` only caches on + // a successful parse, so fixing `metric-views.json` heals on the next + // request. Full reason → telemetry only. + let registry: Record; + try { + registry = await getMetricRegistry(this._queriesDir); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn(req, "Failed to load metric registry: %s", reason); + event?.setContext("analytics", { + metric_registry_load_error: reason, + }); + res.status(503).json({ + error: "Metric registry not available", + code: "METRIC_REGISTRY_LOAD_FAILED", + }); + return; + } + + // Own-property lookup: never resolve `key` to an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …). + // `getMetricRegistry` returns a null-prototype map, but gate the read here + // too as defense-in-depth — an inherited hit would otherwise bypass the 404 + // below and flow a non-registration value into execution. + const registration = Object.hasOwn(registry, key) + ? registry[key] + : undefined; + if (!registration) { + // Don't echo the user-supplied `key` back in the public response — + // confirming "metric X is not registered" lets a probe enumerate keys by + // elimination. The 404 status stays (useful for tooling); the body is + // generic and detail goes to telemetry only. + event?.setContext("analytics", { unknown_metric_key: key }); + res.status(404).json({ error: "Metric not found" }); + return; + } + + // Validate the body on the canonical error path. `validateMetricRequest` + // throws a `ValidationError` (400) whose message names only field paths, + // never raw values. + let request: ReturnType; + try { + request = validateMetricRequest(req.body ?? {}); + } catch (err) { + if (err instanceof AppKitError) { + res.status(err.statusCode).json({ error: err.message, code: err.code }); + return; + } + event?.setContext("analytics", { + unexpected_error: err instanceof Error ? err.message : String(err), + metric_key: key, + }); + logger.warn( + req, + "Unexpected throw during metric request validation for %s: %s", + key, + err instanceof Error ? err.message : String(err), + ); + res.status(400).json({ error: "Invalid request body" }); + return; + } + + // Lane dispatch. The lane comes from the registration (the entry's + // `executor` in metric-views.json), NOT a URL segment or `.obo.sql` + // filename: an OBO-lane metric runs on-behalf-of the requesting user + // (per-user cache via `asUser(req)`), an SP-lane metric as the app service + // principal (shared cache). Compute the executor + key INSIDE a try (as + // `_handleQueryRoute` does) so `asUser(req)`/`resolveUserId(req)` and + // `deriveMetricExecutorKey`'s missing-identity throw land on the canonical + // 401 envelope instead of surfacing as an uncaught 500 out of the stream. + let executor: AnalyticsPlugin; + let executorKey: string; + try { + const isObo = registration.lane === "obo"; + executor = isObo ? this.asUser(req) : this; + executorKey = deriveMetricExecutorKey({ + lane: registration.lane, + userIdentity: isObo ? this.resolveUserId(req) : undefined, + }); + } catch (err) { + if (err instanceof AppKitError) { + res.status(err.statusCode).json({ error: err.message, code: err.code }); + return; + } + throw err; + } + + // Cache key. Composed over the canonicalized args (sorted measures/ + // dimensions, stable-sorted predicates, grain, timeDimension, limit) plus + // the `executorKey` — `"sp"` shares the cache across all users, a per-user + // identity hash isolates OBO callers. Delivery is JSON-only in v1 (the + // route always routes through `_executeJsonArrayPath`), so the key salts on + // a stable "JSON_ARRAY" constant rather than `request.format`: two calls + // that deliver identically must not fork the cache on an unused format + // field. + const cacheConfig = { + ...queryDefaults.cache, + cacheKey: composeMetricCacheKey({ + metricKey: key, + source: registration.source, + measures: request.measures, + dimensions: request.dimensions, + timeGrain: request.timeGrain, + timeDimension: request.timeDimension, + filter: request.filter, + format: "JSON_ARRAY", + executorKey, + limit: request.limit, + }), + }; + + // Cache/retry/timeout scoped to the SQL execution itself (inner `execute`) + // so the warehouse-readiness phase isn't retried and the generator value + // never leaks into the cache. + const sqlConfig: PluginExecuteConfig = { + ...queryDefaults, + cache: cacheConfig, + }; + + // Outer stream: no cache/retry — `executeStream` would otherwise wrap the + // generator factory and cache the generator object itself. Telemetry + + // trace context still apply. + const streamExecutionSettings: StreamExecutionSettings = { + default: { + cache: { enabled: false }, + retry: { enabled: false }, + }, + }; + + const startupTimeoutMs = + this.config.warehouseStartupTimeoutMs ?? + DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS; + const autoStartWarehouse = this.config.autoStartWarehouse ?? true; + + const self = this; + + await executor.executeStream( + res, + async function* ( + signal, + ): AsyncGenerator { + const workspaceClient = getWorkspaceClient(); + const warehouseId = await getWarehouseId(); + + // Stream warehouse-readiness updates as SSE events, then run SQL. + const readinessUpdates = streamCallbacks( + (emit) => + self.SQLClient.ensureWarehouseRunning( + workspaceClient, + warehouseId, + { + signal, + timeoutMs: startupTimeoutMs, + autoStart: autoStartWarehouse, + onStatus: emit, + }, + ), + ); + for await (const update of readinessUpdates) { + yield { + type: "warehouse_status", + status: { + state: update.state as WarehouseStatus["state"], + elapsedMs: update.elapsedMs, + }, + }; + } + + // `execute()` reduces a thrown error to `{ status, message }`, + // dropping the rich fields (`errorCode`, `clientMessage`). Capture the + // original here so we can re-throw it intact — the SSE error path + // (`StreamManager`) reads `errorCode`/`clientMessage` off it. + let originalError: unknown; + const sqlResult = await executor.execute( + async (sig) => { + try { + const { statement, parameters } = buildMetricSql( + registration, + request, + ); + const processedParams = + await self.queryProcessor.processQueryParams( + statement, + Object.keys(parameters).length > 0 ? parameters : undefined, + ); + // Reuse the query route's JSON delivery: INLINE JSON_ARRAY with + // an ARROW_STREAM-inline fallback, returning plain rows in a + // `result` message — byte-identical envelope to `/query`. + return await self._executeJsonArrayPath( + executor, + statement, + processedParams, + sig, + ); + } catch (err) { + originalError = err; + throw err; + } + }, + { default: sqlConfig }, + executorKey, + ); + + if (!sqlResult.ok) { + const msg = sqlResult.message; + const lower = msg.toLowerCase(); + if ( + lower.includes("operation was aborted") || + lower.includes("the request was aborted") || + lower.includes("statement was canceled") + ) { + const err = new DOMException( + lower.includes("canceled") ? msg : "The operation was aborted.", + "AbortError", + ); + throw err; + } + // Re-throw the original error so its structured `errorCode` and + // sanitized `clientMessage` survive to the SSE error payload. Fall + // back to a generic statement failure only if it wasn't an + // AppKitError. + if (originalError instanceof AppKitError) { + throw originalError; + } + const inner = msg.startsWith("Statement failed: ") + ? msg.slice("Statement failed: ".length) + : msg; + throw ExecutionError.statementFailed(inner); + } + + yield sqlResult.data as AnalyticsStreamMessage; + }, + streamExecutionSettings, + executorKey, + ); + } + /** * JSON_ARRAY SSE path. Delegates the disposition/format fallback to * {@link deliverJsonResult} (INLINE JSON_ARRAY → on `needs-arrow-inline`, diff --git a/packages/appkit/src/plugins/analytics/metric.ts b/packages/appkit/src/plugins/analytics/metric.ts new file mode 100644 index 00000000..3e8a6f47 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/metric.ts @@ -0,0 +1 @@ +export * from "./mv"; diff --git a/packages/appkit/src/plugins/analytics/mv/cache.ts b/packages/appkit/src/plugins/analytics/mv/cache.ts new file mode 100644 index 00000000..45de6377 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/cache.ts @@ -0,0 +1,71 @@ +import { createHash } from "node:crypto"; +import { AuthenticationError } from "../../../errors"; +import type { MetricFilter, MetricLane, MetricPredicate } from "../types"; +import { sortFilterChildren } from "./formatters"; +import type { MetricCacheKeyInput } from "./types"; + +export function composeMetricCacheKey(input: MetricCacheKeyInput): string[] { + const sortedMeasures = [...input.measures].sort(); + const sortedDimensions = [...(input.dimensions ?? [])].sort(); + const filterFingerprint = + input.filter !== undefined ? canonicalizeFilter(input.filter) : "_"; + // `timeDimension` only changes the SQL when `timeGrain` is set (see + // renderDimensionClause); keying on it otherwise would fork the cache on a + // no-op field. + const timeDimensionPart = + input.timeGrain != null ? (input.timeDimension ?? "_") : "_"; + return [ + "metric", + input.metricKey, + input.source, + input.format, + sortedMeasures.join(","), + sortedDimensions.join(","), + input.timeGrain ?? "_", + timeDimensionPart, + filterFingerprint, + typeof input.limit === "number" ? String(input.limit) : "_", + input.executorKey, + ]; +} + +export function deriveMetricExecutorKey(input: { + lane: MetricLane; + userIdentity?: string | null; +}): string { + if (input.lane === "sp") { + return "sp"; + } + const identity = input.userIdentity?.trim(); + if (!identity) { + throw AuthenticationError.missingUserId(); + } + return createHash("sha256").update(identity).digest("hex"); +} + +function canonicalizeFilter(node: MetricFilter): string { + if (node === null || typeof node !== "object") { + return "_"; + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + return `${groupKey}()`; + } + + const sorted = sortFilterChildren(children); + const childFingerprints = sorted.map(canonicalizeFilter); + return `${groupKey}(${childFingerprints.join(",")})`; + } + + const p = node as MetricPredicate; + const valuesPart = (p.values ?? []).map((v) => [typeof v, v]); + return `p(${JSON.stringify([p.member, p.operator, valuesPart])})`; +} diff --git a/packages/appkit/src/plugins/analytics/mv/constants.ts b/packages/appkit/src/plugins/analytics/mv/constants.ts new file mode 100644 index 00000000..3676c80b --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/constants.ts @@ -0,0 +1,122 @@ +import path from "node:path"; +import type { MetricFilterOperatorName, MetricLane } from "../types"; + +/** + * Default queries directory. Mirrors `AppManager`'s + * `path.resolve(process.cwd(), "config/queries")` so dev mode and production + * share a single source of truth for where metric config lives. Exported so + * `AnalyticsPlugin` can default `config.queriesDir` to the same path. + */ +export const QUERIES_DIR = path.resolve(process.cwd(), "config/queries"); +export const METRIC_CONFIG_FILE = "metric-views.json"; + +/** + * Measure, dimension, and filter-member names are **column identifiers**: they + * are validated by the shared {@link isValidColumnName} (rejects only control + * characters / newlines) and backtick-quoted via {@link quoteIdentifier} at + * every interpolation point. Quoting — not a narrow ASCII allowlist — is the + * injection boundary, so the runtime accepts the full delimited-identifier + * grammar the type-generator emits from DESCRIBE (hyphens, dots, non-ASCII). + * There is deliberately NO name allowlist: a well-formed-but-unknown column + * falls through to the warehouse and surfaces as a sanitized canonical error. + * + * Time-grain token shape. Unlike the column identifiers above, the grain is + * interpolated as a single-quoted `date_trunc` unit LITERAL (NOT a bind param, + * NOT a delimited identifier) in {@link renderDimensionClause}, so it keeps a + * narrow keyword-shaped gate — that pattern is what keeps a hostile token out + * of the quoted-literal position. + */ +export const TIME_GRAIN_PATTERN = /^[a-z][a-z_]*$/; + +/** + * The exact twelve filter operators allowed at v1. The runtime tuple is the + * server-side source of truth; the client-side type union + * `MetricFilterOperatorName` mirrors these names statically. + */ +export const METRIC_FILTER_OPERATORS = [ + "equals", + "notEquals", + "in", + "notIn", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", + "set", + "notSet", +] as const satisfies readonly MetricFilterOperatorName[]; + +/** + * Maximum AND/OR nesting depth. The PRD documents 8 as a sensible cap — + * enough for any real BI filter UI, low enough that a hostile or malformed + * payload cannot stack-overflow the recursive validator or translator. + * + * The depth count is the number of nested `{ and }` / `{ or }` wrappers + * encountered while descending — leaf predicates do not count toward depth. + */ +export const METRIC_FILTER_MAX_DEPTH = 8; + +/** + * Cardinality caps on user-controlled arrays. Closes the recurring + * `unbounded-request-parameters` finding: a hostile caller could otherwise + * send `values: [...10M items...]` and exhaust the validator + the named + * bind-var binding step. The limits below are deliberately generous — higher + * than any real BI UI would emit — so legitimate traffic never trips them. + */ +export const METRIC_MEASURES_MAX = 50; +export const METRIC_DIMENSIONS_MAX = 20; +export const METRIC_FILTER_VALUES_MAX = 1000; +export const METRIC_LIMIT_MAX = 100_000; + +/** + * Maximum number of children per AND/OR group node. Without this cap a single + * flat group like `{ and: [...10M empty objects...] }` would push tens of + * millions of frames onto the iterative pre-check's stack — OOM before + * validation even gets to Zod. The Zod schema enforces the same cap so the + * rejection point is consistent regardless of which validator catches it + * first. + */ +export const METRIC_FILTER_GROUP_MAX = 100; + +/** Operators that require exactly one value. */ +export const SINGLE_VALUE_OPERATORS = new Set([ + "equals", + "notEquals", + "gt", + "gte", + "lt", + "lte", + "contains", + "notContains", +]); + +/** Operators that require at least one value. */ +export const LIST_VALUE_OPERATORS = new Set([ + "in", + "notIn", +]); + +/** Operators that reject `values` entirely. */ +export const NULL_OPERATORS = new Set([ + "set", + "notSet", +]); + +/** Operators that emit `LIKE` / `NOT LIKE` and require a string value. */ +export const STRING_OPERATORS = new Set([ + "contains", + "notContains", +]); + +/** + * Map an entry's declared `executor` to the internal execution lane: + * - `"user"` → `"obo"` (per-user cache, on-behalf-of) + * - `"app_service_principal"` (default) → `"sp"` (shared cache) + */ +export function laneFromExecutor( + executor: "app_service_principal" | "user", +): MetricLane { + return executor === "user" ? "obo" : "sp"; +} diff --git a/packages/appkit/src/plugins/analytics/mv/formatters.ts b/packages/appkit/src/plugins/analytics/mv/formatters.ts new file mode 100644 index 00000000..57ac74de --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/formatters.ts @@ -0,0 +1,319 @@ +import { type SQLTypeMarker, sql as sqlHelpers } from "shared"; +import { + isValidColumnName, + isValidFqn, + quoteFqnForSql, + quoteIdentifier, +} from "../../../../../shared/src/schemas/metric-fqn"; +import type { + IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, + MetricPredicate, + MetricRegistration, +} from "../types"; +import { + METRIC_FILTER_MAX_DEPTH, + METRIC_FILTER_OPERATORS, + TIME_GRAIN_PATTERN, +} from "./constants"; +import type { FilterRenderState } from "./types"; + +function quoteSafeFqn(fqn: string): string { + if (!isValidFqn(fqn)) { + throw new Error( + `Refusing to build SQL: "${fqn}" is not a valid three-part UC FQN.`, + ); + } + return quoteFqnForSql(fqn); +} + +export function buildMetricSql( + registration: MetricRegistration, + request: IAnalyticsMetricRequest, +): { + statement: string; + parameters: Record; +} { + const quotedSource = quoteSafeFqn(registration.source); + + if (request.measures.length === 0) { + throw new Error("buildMetricSql requires at least one measure."); + } + + for (const m of request.measures) { + if (!isValidColumnName(m)) { + throw new Error( + `Refusing to build SQL: measure "${m}" is not a valid identifier.`, + ); + } + } + + const dimensions = request.dimensions ?? []; + for (const d of dimensions) { + if (!isValidColumnName(d)) { + throw new Error( + `Refusing to build SQL: dimension "${d}" is not a valid identifier.`, + ); + } + } + + const measureClauses = [...request.measures] + .sort() + .map((m) => `MEASURE(${quoteIdentifier(m)}) AS ${quoteIdentifier(m)}`); + + const dimensionClauses = [...dimensions] + .sort() + .map((d) => + renderDimensionClause(d, request.timeGrain, request.timeDimension), + ); + + const selectList = [...measureClauses, ...dimensionClauses].join(", "); + const groupByClause = dimensions.length > 0 ? " GROUP BY ALL" : ""; + + const limitClause = + typeof request.limit === "number" && request.limit > 0 + ? ` LIMIT ${Math.floor(request.limit)}` + : ""; + + const parameters: Record = {}; + let whereClause = ""; + if (request.filter !== undefined) { + const fragment = renderFilter(request.filter, parameters, { + counter: 0, + depth: 0, + }); + if (fragment !== null && fragment.length > 0) { + whereClause = ` WHERE ${fragment}`; + } + } + + const statement = `SELECT ${selectList} FROM ${quotedSource}${whereClause}${groupByClause}${limitClause}`; + return { statement, parameters }; +} + +function renderFilter( + node: MetricFilter, + params: Record, + state: FilterRenderState, +): string | null { + if (node === null || typeof node !== "object") { + throw new Error( + "Refusing to build SQL: filter node must be an object Predicate or { and } / { or } group.", + ); + } + + if ("and" in node || "or" in node) { + const groupKey = "and" in node ? "and" : "or"; + if (state.depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new Error( + `Refusing to build SQL: filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}.`, + ); + } + + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children) || children.length === 0) { + if (groupKey === "or") { + return "1 = 0"; + } + return null; + } + + const sortedChildren = sortFilterChildren(children); + + const fragments: string[] = []; + const childState: FilterRenderState = { + counter: state.counter, + depth: state.depth + 1, + }; + for (const child of sortedChildren) { + const rendered = renderFilter(child, params, childState); + if (rendered != null && rendered.length > 0) { + fragments.push(rendered); + } + } + state.counter = childState.counter; + + if (fragments.length === 0) return null; + if (fragments.length === 1) return fragments[0]; + const joiner = groupKey === "and" ? " AND " : " OR "; + return `(${fragments.join(joiner)})`; + } + + const predicate = node as MetricPredicate; + + if (!isValidColumnName(predicate.member)) { + throw new Error( + `Refusing to build SQL: filter member "${predicate.member}" is not a valid identifier.`, + ); + } + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + throw new Error( + `Refusing to build SQL: unknown filter operator "${predicate.operator}".`, + ); + } + + return renderPredicate(predicate, params, state); +} + +export function sortFilterChildren( + children: ReadonlyArray, +): MetricFilter[] { + const indexed = children.map((child, idx) => { + let key: string; + let isPredicate: boolean; + if ( + child !== null && + typeof child === "object" && + !("and" in child) && + !("or" in child) + ) { + const p = child as MetricPredicate; + key = JSON.stringify([p.member, p.operator]); + isPredicate = true; + } else { + key = ""; + isPredicate = false; + } + return { child, idx, key, isPredicate }; + }); + + indexed.sort((a, b) => { + if (a.isPredicate && !b.isPredicate) return -1; + if (!a.isPredicate && b.isPredicate) return 1; + if (a.isPredicate && b.isPredicate) { + if (a.key < b.key) return -1; + if (a.key > b.key) return 1; + } + return a.idx - b.idx; + }); + + return indexed.map((entry) => entry.child); +} + +function renderPredicate( + predicate: MetricPredicate, + params: Record, + state: FilterRenderState, +): string { + const col = quoteIdentifier(predicate.member); + const op = predicate.operator; + const values = predicate.values ?? []; + + switch (op) { + case "equals": + return `${col} = ${bindValue(values[0], params, state)}`; + case "notEquals": + return `${col} <> ${bindValue(values[0], params, state)}`; + case "gt": + return `${col} > ${bindValue(values[0], params, state)}`; + case "gte": + return `${col} >= ${bindValue(values[0], params, state)}`; + case "lt": + return `${col} < ${bindValue(values[0], params, state)}`; + case "lte": + return `${col} <= ${bindValue(values[0], params, state)}`; + case "in": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} IN (${placeholders.join(", ")})`; + } + case "notIn": { + const placeholders = values.map((v) => bindValue(v, params, state)); + return `${col} NOT IN (${placeholders.join(", ")})`; + } + case "contains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "contains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} LIKE ${bindLikeValue(raw, params, state)}`; + } + case "notContains": { + const raw = values[0]; + if (typeof raw !== "string") { + throw new Error( + `Refusing to build SQL: filter operator "notContains" requires a string value (got ${typeof raw}).`, + ); + } + return `${col} NOT LIKE ${bindLikeValue(raw, params, state)}`; + } + case "set": + return `${col} IS NOT NULL`; + case "notSet": + return `${col} IS NULL`; + default: { + const _exhaustive: never = op; + throw new Error( + `Refusing to build SQL: unhandled filter operator "${_exhaustive as string}".`, + ); + } + } +} + +function bindValue( + value: string | number | undefined, + params: Record, + state: FilterRenderState, +): string { + if (value === undefined) { + throw new Error( + "Refusing to build SQL: filter predicate is missing a required value.", + ); + } + const name = `f_${state.counter}`; + state.counter += 1; + if (typeof value === "number") { + params[name] = sqlHelpers.number(value); + } else if (typeof value === "string") { + params[name] = sqlHelpers.string(value); + } else { + throw new Error( + `Refusing to build SQL: filter value must be a string or number (got ${typeof value}).`, + ); + } + return `:${name}`; +} + +function bindLikeValue( + value: string, + params: Record, + state: FilterRenderState, +): string { + const name = `f_${state.counter}`; + state.counter += 1; + params[name] = sqlHelpers.string(`%${value}%`); + return `:${name}`; +} + +function renderDimensionClause( + dim: string, + timeGrain: string | undefined, + timeDimension: string | undefined, +): string { + if (timeGrain != null && dim === timeDimension) { + if (!isValidColumnName(dim)) { + throw new Error( + `Refusing to build SQL: timeDimension "${dim}" is not a valid identifier.`, + ); + } + if (!TIME_GRAIN_PATTERN.test(timeGrain)) { + throw new Error( + `Refusing to build SQL: timeGrain "${timeGrain}" is not a valid grain token.`, + ); + } + const quoted = quoteIdentifier(dim); + return `date_trunc('${timeGrain}', ${quoted}) AS ${quoted}`; + } + return quoteIdentifier(dim); +} diff --git a/packages/appkit/src/plugins/analytics/mv/index.ts b/packages/appkit/src/plugins/analytics/mv/index.ts new file mode 100644 index 00000000..de5eec8f --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/index.ts @@ -0,0 +1,9 @@ +export { composeMetricCacheKey, deriveMetricExecutorKey } from "./cache"; +export { QUERIES_DIR } from "./constants"; +export { buildMetricSql } from "./formatters"; +export { + __resetMetricRegistryCache, + getMetricRegistry, + loadMetricRegistry, +} from "./registry"; +export { validateMetricRequest } from "./schemas"; diff --git a/packages/appkit/src/plugins/analytics/mv/registry.ts b/packages/appkit/src/plugins/analytics/mv/registry.ts new file mode 100644 index 00000000..190fd1a6 --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/registry.ts @@ -0,0 +1,192 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +// Canonical metric-source schema — the single source of truth for +// `metric-views.json`. Imported from the shared source directly (matching the +// type-generator's runtime, which pulls the zod-free `metric-fqn.ts` from the +// same tree) so the runtime and the generated JSON schema validate identically. +import { metricSourceSchema } from "../../../../../shared/src/schemas/metric-source"; +import { createLogger } from "../../../logging/logger"; +import type { MetricRegistration } from "../types"; +import { laneFromExecutor, METRIC_CONFIG_FILE, QUERIES_DIR } from "./constants"; +import type { RegistryCacheSignature } from "./types"; + +const logger = createLogger("analytics:metric"); + +/** + * Read and validate `config/queries/metric-views.json` into a metric registry. + * + * Async and stateless — registration is a pure config parse with no warehouse + * round-trip, no `DESCRIBE`, and no build-time metadata bundle. The single + * `metricViews` map makes keys unique by construction, so there is no + * cross-lane duplicate-key check. Async I/O (rather than `readFileSync`) keeps + * the event loop free: metric views are already heavier than a plain `.sql` + * query on the warehouse side, so the SDK layer must not add a blocking read + * on top. Caching + mtime-revalidation is layered on top by + * {@link getMetricRegistry} — this function always hits disk. + * + * Returns an empty registry when the file is absent: the metric-view path is + * additive and dormant until an app opts in by adding the config. A malformed + * file (unreadable, invalid JSON, or schema violation) throws — the caller + * surfaces a 503 rather than masking a broken deployment as a 404 for every + * key. The failure is NOT cached (see {@link getMetricRegistry}), so fixing the + * file heals on the next request. + */ +export async function loadMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let raw: string; + try { + raw = await fs.readFile(metricPath, "utf8"); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return Object.create(null); + } + throw err; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `Failed to parse metric-views.json at ${metricPath}: ${(err as Error).message}`, + ); + } + + const result = metricSourceSchema.safeParse(parsed); + if (!result.success) { + const issues = result.error.issues + .map((i) => `${i.path.join(".")}: ${i.message}`) + .join("; "); + throw new Error(`Invalid metric-views.json at ${metricPath}: ${issues}`); + } + + // Null-prototype map so a metric key that collides with an inherited + // `Object.prototype` member (`__proto__`, `constructor`, `toString`, …) + // cannot resolve to a truthy non-registration at the `registry[key]` read + // site and slip past the unknown-key 404. Keys are still grammar-gated by + // `metricKeySchema` (identifier shape), but the null prototype removes the + // whole class of inherited-property lookups as a boundary. + const registry: Record = Object.create(null); + for (const [key, entry] of Object.entries(result.data.metricViews ?? {})) { + registry[key] = { + key, + source: entry.source, + lane: laneFromExecutor(entry.executor), + }; + } + + logger.debug( + "Loaded metric registry: %d entry(ies)", + Object.keys(registry).length, + ); + return registry; +} + +/** + * Module-level registry cache, keyed by the resolved queries directory. + * + * Keyed by DIR (not by plugin instance) because the registry is a pure + * function of the config file at that path — warehouse-independent, and two + * `AnalyticsPlugin` instances pointed at the same `config/queries/` MUST see + * the same registry. Instance state would parse the identical file twice and + * risk divergence; a dir-keyed module cache shares one parse. + */ +const metricRegistryCache = new Map< + string, + { + signature: RegistryCacheSignature; + registry: Record; + } +>(); + +/** + * Clear the module-level registry cache. + * + * @internal FOR TESTS ONLY. The cache is keyed by directory and lives for the + * process; production never needs to clear it (a changed file is picked up via + * the stat signature). Tests that reuse a directory — or assert cold-load + * behavior — call this in `beforeEach`/`afterEach` so isolation is intentional + * rather than relying on each test happening to use a unique temp dir. + */ +export function __resetMetricRegistryCache(): void { + metricRegistryCache.clear(); +} + +/** + * Resolve the metric registry for `queriesDir`, re-reading + re-parsing only + * when `metric-views.json` has changed since the cached copy. + * + * Behaves like the sibling `.sql` query path (which re-reads per request) but + * cheaper: the steady-state cost is a single async `stat`, and the read + + * `JSON.parse` + zod validation are skipped when the file's + * `(ctimeMs, mtimeMs, size)` signature is unchanged. This delivers the agreed + * semantics without the old permanent memo: + * + * - **Hot-reload** — editing a working config bumps `mtimeMs`, so the next + * request re-parses and serves the new registry with no server restart. + * - **Self-heal** — a load failure is NOT cached (we only populate the cache + * on a successful parse), so a fixed config is picked up on the next + * request instead of latching a 503 forever. + * - **Dormant** — an absent file `stat`s as `ENOENT` → empty registry; adding + * the file later is picked up on the next request. + * + * Concurrency: two simultaneous cold requests may both `stat`+read before + * either populates the cache. That is a harmless redundant read of the same + * file (the sibling `.sql` path does not single-flight either), so no lock is + * taken. + * + * @throws Propagates {@link loadMetricRegistry}'s throw on a malformed file so + * the route can surface a 503; the cache is left untouched on failure. + */ +export async function getMetricRegistry( + queriesDir: string = QUERIES_DIR, +): Promise> { + const metricPath = path.join(queriesDir, METRIC_CONFIG_FILE); + + let signature: RegistryCacheSignature | null; + try { + const stats = await fs.stat(metricPath); + signature = { + ctimeMs: stats.ctimeMs, + mtimeMs: stats.mtimeMs, + size: stats.size, + }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + // Absent file → dormant. Drop any stale cache entry (the file may have + // been deleted) and return an empty registry without touching the cache. + metricRegistryCache.delete(queriesDir); + signature = null; + } else { + // Any other stat error (EACCES / EIO / ELOOP / …) is deliberately fatal + // for THIS request → the route surfaces a 503, consistent with the + // malformed-config → 503 path. It is not latched: with the self-heal + // design a transient error clears on the next request, which is strictly + // better than the old memo that could latch-and-serve-stale. + throw err; + } + } + + if (signature === null) { + return Object.create(null); + } + + const cached = metricRegistryCache.get(queriesDir); + if ( + cached !== undefined && + cached.signature.ctimeMs === signature.ctimeMs && + cached.signature.mtimeMs === signature.mtimeMs && + cached.signature.size === signature.size + ) { + return cached.registry; + } + + // Cold or stale: re-read + re-parse. Cache ONLY on success so a malformed + // file never latches — the next request re-attempts and heals. + const registry = await loadMetricRegistry(queriesDir); + metricRegistryCache.set(queriesDir, { signature, registry }); + return registry; +} diff --git a/packages/appkit/src/plugins/analytics/mv/schemas.ts b/packages/appkit/src/plugins/analytics/mv/schemas.ts new file mode 100644 index 00000000..0f00dcad --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/schemas.ts @@ -0,0 +1,356 @@ +import { z } from "zod"; +import { isValidColumnName } from "../../../../../shared/src/schemas/metric-fqn"; +import { ValidationError } from "../../../errors"; +import type { + IAnalyticsMetricRequest, + MetricFilter, + MetricFilterOperatorName, + MetricPredicate, +} from "../types"; +import { normalizeAnalyticsFormat } from "../types"; +import { + LIST_VALUE_OPERATORS, + METRIC_DIMENSIONS_MAX, + METRIC_FILTER_GROUP_MAX, + METRIC_FILTER_MAX_DEPTH, + METRIC_FILTER_OPERATORS, + METRIC_FILTER_VALUES_MAX, + METRIC_LIMIT_MAX, + METRIC_MEASURES_MAX, + NULL_OPERATORS, + SINGLE_VALUE_OPERATORS, + STRING_OPERATORS, + TIME_GRAIN_PATTERN, +} from "./constants"; + +/** A leaf predicate: `{ member, operator, values? }`, no extra keys. */ +const filterPredicateSchema: z.ZodType = z + .object({ + member: z + .string() + .min(1, { message: "filter predicate 'member' cannot be empty" }) + .refine(isValidColumnName, { + message: + "filter predicate 'member' contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + operator: z.string().min(1, { + message: "filter predicate 'operator' cannot be empty", + }) as z.ZodType, + values: z + .array(z.union([z.string(), z.number()])) + .max(METRIC_FILTER_VALUES_MAX, { + message: `filter predicate 'values' length exceeds the maximum of ${METRIC_FILTER_VALUES_MAX}`, + }) + .optional(), + }) + .strict(); + +/** Recursive filter: a predicate leaf or an `{ and }` / `{ or }` group. */ +const filterSchema: z.ZodType = z.lazy(() => + z.union([ + filterPredicateSchema, + z + .object({ + and: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'and' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + z + .object({ + or: z.array(filterSchema).max(METRIC_FILTER_GROUP_MAX, { + message: `filter 'or' group exceeds the maximum of ${METRIC_FILTER_GROUP_MAX} children`, + }), + }) + .strict(), + ]), +); + +const metricRequestSchema = z + .object({ + measures: z + .array( + z + .string() + .min(1, "measure name cannot be empty") + .refine(isValidColumnName, { + message: + "measure name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) + .min(1, "at least one measure is required") + .max(METRIC_MEASURES_MAX, { + message: `measures length exceeds the maximum of ${METRIC_MEASURES_MAX}`, + }), + dimensions: z + .array( + z + .string() + .min(1, "dimension name cannot be empty") + .refine(isValidColumnName, { + message: + "dimension name contains a character that cannot be used in a SQL identifier (control character or newline)", + }), + ) + .max(METRIC_DIMENSIONS_MAX, { + message: `dimensions length exceeds the maximum of ${METRIC_DIMENSIONS_MAX}`, + }) + .optional(), + filter: filterSchema.optional(), + // Grammar-shaped bucketing grain, applied to `timeDimension` via + // `date_trunc`. The token is validated for safety here; the grain literal + // is interpolated (single-quoted, never a bind param) in + // `renderDimensionClause`, so this pattern gate is the security boundary. + timeGrain: z + .string() + .min(1, { message: "timeGrain cannot be empty" }) + .regex(TIME_GRAIN_PATTERN, { + message: "timeGrain must match /^[a-z][a-z_]*$/", + }) + .optional(), + // The single dimension `timeGrain` applies to via `date_trunc`. A column + // identifier (backtick-quoted at interpolation), so it accepts the full + // delimited-identifier grammar. Cross-field rules in `superRefine`: + // required when `timeGrain` is set, and must be one of `dimensions`. + timeDimension: z + .string() + .min(1, { message: "timeDimension cannot be empty" }) + .refine(isValidColumnName, { + message: + "timeDimension contains a character that cannot be used in a SQL identifier (control character or newline)", + }) + .optional(), + limit: z + .number() + .int({ message: "limit must be an integer" }) + .positive({ message: "limit must be positive" }) + .max(METRIC_LIMIT_MAX, { + message: `limit exceeds the maximum of ${METRIC_LIMIT_MAX}`, + }) + .optional(), + format: z.enum(["JSON_ARRAY", "ARROW_STREAM", "JSON", "ARROW"]).optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.filter != null) { + validateFilterTree(value.filter, ctx, ["filter"], 0); + } + + const seen = new Set(); + const collided = new Set(); + for (const name of [...value.measures, ...(value.dimensions ?? [])]) { + if (seen.has(name)) { + collided.add(name); + } + seen.add(name); + } + if (collided.size > 0) { + ctx.addIssue({ + code: "custom", + message: + "measures and dimensions must be unique across both lists (a name cannot repeat, nor appear as both a measure and a dimension)", + path: ["measures"], + }); + } + + if ( + value.format != null && + normalizeAnalyticsFormat(value.format) !== "JSON_ARRAY" + ) { + ctx.addIssue({ + code: "custom", + message: + "format: only JSON_ARRAY is supported on the metric route at v1 (ARROW_STREAM is not yet implemented)", + path: ["format"], + }); + } + + if (value.timeGrain != null && value.timeDimension == null) { + ctx.addIssue({ + code: "custom", + message: "timeDimension is required when timeGrain is set", + path: ["timeDimension"], + }); + } + if ( + value.timeDimension != null && + !(value.dimensions ?? []).includes(value.timeDimension) + ) { + ctx.addIssue({ + code: "custom", + message: "timeDimension must be one of dimensions", + path: ["timeDimension"], + }); + } + }) as z.ZodType; + +function validateFilterTree( + node: MetricFilter, + ctx: z.RefinementCtx, + path: Array, + depth: number, +): void { + if (node === null || typeof node !== "object") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: "filter node must be a Predicate or { and } / { or } group", + }); + return; + } + + if ("and" in node || "or" in node) { + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path, + message: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }); + return; + } + + const groupKey = "and" in node ? "and" : "or"; + const children = ( + node as { and?: ReadonlyArray } & { + or?: ReadonlyArray; + } + )[groupKey]; + + if (!Array.isArray(children)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, groupKey], + message: `filter ${groupKey} group must be an array of predicates or nested groups`, + }); + return; + } + + if (groupKey === "or" && children.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "or"], + message: "filter 'or' group must contain at least one predicate", + }); + return; + } + + children.forEach((child, idx) => { + validateFilterTree(child, ctx, [...path, groupKey, idx], depth + 1); + }); + return; + } + + const predicate = node as MetricPredicate; + + if ( + !METRIC_FILTER_OPERATORS.includes( + predicate.operator as MetricFilterOperatorName, + ) + ) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "operator"], + message: `filter operator "${predicate.operator}" is not one of: ${METRIC_FILTER_OPERATORS.join(", ")}`, + }); + return; + } + + const op = predicate.operator; + const values = predicate.values; + const valuesLen = values?.length ?? 0; + + if (NULL_OPERATORS.has(op)) { + if (values != null && valuesLen > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" must not carry values`, + }); + } + } else if (SINGLE_VALUE_OPERATORS.has(op)) { + if (valuesLen !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires exactly one value (got ${valuesLen})`, + }); + } + } else if (LIST_VALUE_OPERATORS.has(op)) { + if (valuesLen < 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires at least one value`, + }); + } + } + + if (STRING_OPERATORS.has(op) && valuesLen > 0) { + const v = predicate.values?.[0]; + if (typeof v !== "string") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [...path, "values"], + message: `filter operator "${op}" requires a string value (got ${typeof v})`, + }); + } + } +} + +function preCheckFilterDepth(filter: unknown): void { + if (filter == null || typeof filter !== "object") return; + const stack: Array<[unknown, number]> = [[filter, 0]]; + while (stack.length > 0) { + const popped = stack.pop(); + if (popped === undefined) continue; + const [node, depth] = popped; + if (node == null || typeof node !== "object") continue; + const obj = node as Record; + for (const groupKey of ["and", "or"] as const) { + const children = obj[groupKey]; + if (!Array.isArray(children)) continue; + if (children.length > METRIC_FILTER_GROUP_MAX) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter ${groupKey} group has ${children.length} children; the maximum is ${METRIC_FILTER_GROUP_MAX}`, + }, + }, + ); + } + if (depth + 1 > METRIC_FILTER_MAX_DEPTH) { + throw new ValidationError( + "Invalid metric request body (fields: filter)", + { + context: { + reason: `filter AND/OR nesting exceeds the maximum depth of ${METRIC_FILTER_MAX_DEPTH}`, + }, + }, + ); + } + for (const child of children) { + stack.push([child, depth + 1]); + } + } + } +} + +export function validateMetricRequest(body: unknown): IAnalyticsMetricRequest { + if (body != null && typeof body === "object") { + preCheckFilterDepth((body as { filter?: unknown }).filter); + } + const result = metricRequestSchema.safeParse(body); + if (!result.success) { + const fieldPaths = result.error.issues + .map((i) => i.path.join(".") || "(root)") + .join(", "); + throw new ValidationError( + fieldPaths.length > 0 + ? `Invalid metric request body (fields: ${fieldPaths})` + : "Invalid metric request body", + { context: { issues: result.error.issues } }, + ); + } + return result.data; +} diff --git a/packages/appkit/src/plugins/analytics/mv/types.ts b/packages/appkit/src/plugins/analytics/mv/types.ts new file mode 100644 index 00000000..be33cf7d --- /dev/null +++ b/packages/appkit/src/plugins/analytics/mv/types.ts @@ -0,0 +1,25 @@ +import type { MetricFilter } from "../types"; + +export interface RegistryCacheSignature { + ctimeMs: number; + mtimeMs: number; + size: number; +} + +export interface FilterRenderState { + counter: number; + depth: number; +} + +export interface MetricCacheKeyInput { + metricKey: string; + source: string; + measures: string[]; + dimensions?: string[]; + timeGrain?: string; + timeDimension?: string; + filter?: MetricFilter; + format: string; + executorKey: string; + limit?: number; +} diff --git a/packages/appkit/src/plugins/analytics/tests/analytics.test.ts b/packages/appkit/src/plugins/analytics/tests/analytics.test.ts index 35677792..aff24614 100644 --- a/packages/appkit/src/plugins/analytics/tests/analytics.test.ts +++ b/packages/appkit/src/plugins/analytics/tests/analytics.test.ts @@ -90,18 +90,23 @@ describe("Analytics Plugin", () => { }); describe("injectRoutes", () => { - test("should register single POST route for queries", () => { + test("should register the query and metric POST routes", () => { const plugin = new AnalyticsPlugin(config); const { router } = createMockRouter(); plugin.injectRoutes(router); - // Only 1 POST route - asUser is determined by .obo.sql file convention - expect(router.post).toHaveBeenCalledTimes(1); + // Two POST routes: the SQL query route (asUser determined by the + // .obo.sql file convention) and the metric-view route. + expect(router.post).toHaveBeenCalledTimes(2); expect(router.post).toHaveBeenCalledWith( "/query/:query_key", expect.any(Function), ); + expect(router.post).toHaveBeenCalledWith( + "/metric/:key", + expect.any(Function), + ); }); test("/query/:query_key should return 400 when query_key is missing", async () => { diff --git a/packages/appkit/src/plugins/analytics/tests/metric.test.ts b/packages/appkit/src/plugins/analytics/tests/metric.test.ts new file mode 100644 index 00000000..45ed380f --- /dev/null +++ b/packages/appkit/src/plugins/analytics/tests/metric.test.ts @@ -0,0 +1,2243 @@ +import { mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { + createMockRequest, + createMockResponse, + createMockRouter, + mockServiceContext, + setupDatabricksEnv, +} from "@tools/test-helpers"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { ServiceContext } from "../../../context/service-context"; +import { AuthenticationError } from "../../../errors"; +import { AnalyticsPlugin } from "../analytics"; +import { + __resetMetricRegistryCache, + buildMetricSql, + composeMetricCacheKey, + deriveMetricExecutorKey, + getMetricRegistry, + loadMetricRegistry, + validateMetricRequest, +} from "../metric"; +import type { + IAnalyticsConfig, + MetricFilter, + MetricRegistration, +} from "../types"; + +// Mirror the analytics.test.ts CacheManager mock so the inner `execute`'s +// cache interceptor is a no-op pass-through (each request re-executes). +const { mockCacheStore, mockCacheInstance } = vi.hoisted(() => { + const store = new Map(); + const generateKey = (parts: unknown[], userKey: string): string => { + const { createHash } = require("node:crypto"); + const serialized = JSON.stringify([userKey, ...parts]); + return createHash("sha256").update(serialized).digest("hex"); + }; + const instance = { + get: vi.fn(), + set: vi.fn(), + delete: vi.fn(), + getOrExecute: vi.fn( + async (key: unknown[], fn: () => Promise, userKey: string) => { + const cacheKey = generateKey(key, userKey); + if (store.has(cacheKey)) return store.get(cacheKey); + const result = await fn(); + store.set(cacheKey, result); + return result; + }, + ), + generateKey: vi.fn((parts: unknown[], userKey: string) => + generateKey(parts, userKey), + ), + }; + return { mockCacheStore: store, mockCacheInstance: instance }; +}); + +vi.mock("../../../cache", () => ({ + CacheManager: { + getInstanceSync: vi.fn(() => mockCacheInstance), + }, +})); + +// Temp dirs created by `registryDir` / `writeRegistry`, cleaned up after each +// test. Using real files (via the test-only `queriesDir` config) exercises the +// actual stat → read → cache path in `getMetricRegistry` rather than poking +// private plugin state. +const tempRegistryDirs: string[] = []; + +/** + * Write a `metric-views.json` into a fresh temp dir and return the dir, for use + * as `new AnalyticsPlugin({ ...config, queriesDir })`. Accepts the internal + * `MetricRegistration` shape (matching the old `setRegistry` helper) and maps + * each entry's `lane` back to the config's `executor` field. + */ +function registryDir(registry: Record): string { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + const metricViews: Record = {}; + for (const [key, reg] of Object.entries(registry)) { + metricViews[key] = { + source: reg.source, + executor: reg.lane === "obo" ? "user" : "app_service_principal", + }; + } + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + return dir; +} + +/** + * Overwrite the `metric-views.json` in an existing temp dir (for hot-reload / + * self-heal tests). `raw` lets a test write deliberately malformed content. + */ +function writeRegistry( + dir: string, + content: Record | string, +): void { + const body = + typeof content === "string" + ? content + : JSON.stringify({ + metricViews: Object.fromEntries( + Object.entries(content).map(([key, reg]) => [ + key, + { + source: reg.source, + executor: reg.lane === "obo" ? "user" : "app_service_principal", + }, + ]), + ), + }); + writeFileSync(path.join(dir, "metric-views.json"), body); +} + +describe("analytics metric route (Phase 1)", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + __resetMetricRegistryCache(); + while (tempRegistryDirs.length > 0) { + const dir = tempRegistryDirs.pop(); + if (dir) rmSync(dir, { recursive: true, force: true }); + } + }); + + describe("injectRoutes", () => { + test("registers POST /metric/:key alongside /query", () => { + const plugin = new AnalyticsPlugin(config); + const { router } = createMockRouter(); + + plugin.injectRoutes(router); + + expect(router.post).toHaveBeenCalledWith( + "/metric/:key", + expect.any(Function), + ); + }); + }); + + // ── Identifier safety — the primary security test. Measure/dimension names + // are backtick-quoted (not narrow-gated) at interpolation, so an injection- + // shaped name is NEUTRALIZED by quoting rather than rejected: it becomes an + // inert (if nonexistent) column the warehouse resolves away. Only a name + // that cannot be safely quoted at all — one containing a control character + // or newline — is refused. + describe("buildMetricSql identifier safety (quoting)", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("neutralizes an injection-shaped measure by quoting (no breakout)", () => { + // `arr; DROP TABLE users` has no control chars, so it is a valid (if + // nonexistent) column name: quoted whole, the `;` and `DROP` are inert + // inside the backtick-delimited identifier — not separate statements. + const { statement } = buildMetricSql(registration, { + measures: ["arr; DROP TABLE users"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr; DROP TABLE users`) AS `arr; DROP TABLE users` FROM `cat`.`sch`.`revenue_metrics`", + ); + }); + + test("neutralizes a backtick in a measure by doubling it", () => { + // The one real breakout char is the backtick; quoteIdentifier doubles it + // so it cannot close the identifier early. + const { statement } = buildMetricSql(registration, { + measures: ["arr`"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr```) AS `arr``` FROM `cat`.`sch`.`revenue_metrics`", + ); + }); + + test("throws for a measure containing a control character (cannot be quoted)", () => { + expect(() => + buildMetricSql(registration, { measures: ["arr\ndrop"] }), + ).toThrow(/not a valid identifier|control character/); + }); + + test("throws when no measures are supplied", () => { + expect(() => buildMetricSql(registration, { measures: [] })).toThrow( + /at least one measure/, + ); + }); + + test("throws for a non-three-part source FQN", () => { + expect(() => + buildMetricSql( + { key: "x", source: "cat.sch", lane: "sp" }, + { measures: ["arr"] }, + ), + ).toThrow(/not a valid three-part UC FQN/); + }); + + test("accepts a hyphenated UC-legal FQN and backtick-quotes it", () => { + // Regression for the grammar divergence: `prod-data` is a legal UC + // object name (hyphens are allowed in quoted identifiers) and passes the + // shared schema + typegen, but the old narrow runtime pattern + // [a-zA-Z0-9_-] anchored per segment REJECTED it — a latent prod break on + // a documented-legal name. The builder now accepts it and quotes every + // segment, so it reaches the warehouse as valid SQL rather than throwing. + const { statement } = buildMetricSql( + { key: "x", source: "prod-data.analytics.revenue", lane: "sp" }, + { measures: ["arr"] }, + ); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr` FROM `prod-data`.`analytics`.`revenue`", + ); + }); + + test("backtick-quoting neutralizes an injection-shaped FQN segment", () => { + // A source carrying a backtick would break out of the quoted identifier + // if interpolated raw; quoteFqnForSql doubles it. (isValidFqn rejects + // most such names first, but the quoting is the actual injection + // boundary — this asserts it, not just the grammar gate.) + const { statement } = buildMetricSql( + { key: "x", source: "cat.sch.re`v", lane: "sp" }, + { measures: ["arr"] }, + ); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`re``v`", + ); + }); + }); + + // ── Measures-only SQL shape. + describe("buildMetricSql SQL shape", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("single measure → SELECT MEASURE(`m`) AS `m` FROM ", () => { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", + ); + expect(parameters).toEqual({}); + }); + + test("multiple measures are sorted for a deterministic SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["revenue", "arr"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue` FROM `cat`.`sch`.`revenue_metrics`", + ); + }); + + test("positive limit appends a floored LIMIT clause", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + limit: 10.9, + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics` LIMIT 10", + ); + }); + }); + + // ── Phase 2: dimensions + GROUP BY ALL. Bare dimensions here; date_trunc + // grain application (via timeDimension) is covered in its own block below. + describe("buildMetricSql dimensions + GROUP BY", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("measures + dimensions → SELECT MEASURE(...), FROM GROUP BY ALL", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("dimensions are sorted for a deterministic SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["segment", "region"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region`, `segment` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("measures sort before dimensions in the SELECT list", () => { + const { statement } = buildMetricSql(registration, { + measures: ["revenue", "arr"], + dimensions: ["region"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, MEASURE(`revenue`) AS `revenue`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("empty dimensions array → no GROUP BY (ungrouped)", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: [], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", + ); + }); + + test("dimensions + limit compose GROUP BY ALL then LIMIT", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region"], + limit: 100, + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL LIMIT 100", + ); + }); + + test("no timeGrain → all dimensions render bare (no date_trunc)", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date", "region"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `order_date`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + expect(statement).not.toContain("date_trunc"); + }); + }); + + // ── Phase 2a: timeGrain + timeDimension → date_trunc on the named column. + // The grain is a grammar-gated single-quoted literal; the column keeps its + // plain alias; other dimensions render bare; GROUP BY ALL is present. + describe("buildMetricSql timeGrain + timeDimension (date_trunc)", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("buckets only the timeDimension via date_trunc; other dimensions bare", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "order_date", + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, date_trunc('month', `order_date`) AS `order_date`, `region` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + expect(statement).toContain( + "date_trunc('month', `order_date`) AS `order_date`", + ); + expect(statement).toContain(" GROUP BY ALL"); + }); + + test("the grain literal is threaded through (day) for the timeDimension", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, date_trunc('day', `order_date`) AS `order_date` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("a grammar-invalid timeGrain throws in the builder (defense-in-depth, even if validation is bypassed)", () => { + // buildMetricSql is exported and may be reached on a path that skips + // validateMetricRequest, so the grain — interpolated into a single-quoted + // date_trunc literal — is re-gated at the interpolation point. A quote- + // breakout payload must be refused by the builder itself. + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month'); DROP TABLE t;--", + timeDimension: "order_date", + }), + ).toThrow(/not a valid grain token/); + }); + }); + + // ── Phase 2: dimension identifier safety. A dimension is backtick-quoted at + // interpolation, so an injection-shaped name is neutralized (inert quoted + // column), and only an unquotable (control-char) name throws. + describe("buildMetricSql dimension identifier safety (quoting)", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + test("neutralizes an injection-shaped dimension by quoting", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region; DROP TABLE users"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region; DROP TABLE users` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("neutralizes a backtick in a dimension by doubling it", () => { + const { statement } = buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region`"], + }); + expect(statement).toBe( + "SELECT MEASURE(`arr`) AS `arr`, `region``` FROM `cat`.`sch`.`revenue_metrics` GROUP BY ALL", + ); + }); + + test("throws for a dimension containing a control character", () => { + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + dimensions: ["region\tbad"], + }), + ).toThrow(/not a valid identifier|control character/); + }); + }); + + // ── Envelope parity — streams warehouse_status* then a `result` message, + // byte-identical to the /query route's JSON SSE path. + describe("_handleMetricRoute SSE envelope", () => { + test("streams warehouse_status then a result message with aliased rows", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn().mockResolvedValue({ + result: { data: [{ arr: 1234 }] }, + }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The measures-only SQL reaches the warehouse. + expect(executeMock).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + statement: + "SELECT MEASURE(`arr`) AS `arr` FROM `cat`.`sch`.`revenue_metrics`", + warehouse_id: "test-warehouse-id", + }), + expect.any(AbortSignal), + ); + + // Same SSE envelope as /query: a `result` event carrying the rows. + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + expect(mockRes.write).toHaveBeenCalledWith( + expect.stringContaining('"data":[{"arr":1234}]'), + ); + expect(mockRes.end).toHaveBeenCalled(); + }); + + test("emits warehouse_status before result for a STARTING warehouse", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn().mockResolvedValue({ + result: { data: [{ arr: 1 }] }, + }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + const warehouseGet = vi + .fn() + .mockResolvedValueOnce({ state: "STARTING" }) + .mockResolvedValueOnce({ state: "RUNNING" }); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + mockReq.serviceWorkspaceClient.warehouses.get = warehouseGet; + mockReq.userWorkspaceClient.warehouses.get = warehouseGet; + const mockRes = createMockResponse(); + + vi.useFakeTimers(); + const handlerPromise = handler(mockReq, mockRes); + await vi.runAllTimersAsync(); + await handlerPromise; + vi.useRealTimers(); + + const eventLines = (mockRes.write as any).mock.calls + .map((call: any[]) => call[0] as string) + .filter((s: string) => s.startsWith("event: ")); + const warehouseIdx = eventLines.findIndex( + (s: string) => s === "event: warehouse_status\n", + ); + const resultIdx = eventLines.findIndex( + (s: string) => s === "event: result\n", + ); + expect(warehouseIdx).toBeGreaterThanOrEqual(0); + expect(resultIdx).toBeGreaterThanOrEqual(0); + expect(warehouseIdx).toBeLessThan(resultIdx); + }); + + test("returns 400 when the body fails structural validation", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: [] }, // empty → fails .min(1) + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(400); + }); + }); + + // ── 503-vs-404 latching + dormancy. + describe("registry latching and dormancy", () => { + test("unknown key against a valid registry → 404 (generic body)", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "nope" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(mockRes.json).toHaveBeenCalledWith({ error: "Metric not found" }); + }); + + test.each(["__proto__", "constructor", "toString", "hasOwnProperty"])( + "inherited Object.prototype key %j → 404, no execution (own-property lookup)", + async (dangerousKey) => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + // A real (own) entry so the registry is populated but does NOT contain + // the dangerous key. Without the own-property guard, `registry[key]` + // would resolve the inherited prototype member and slip past the 404. + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: dangerousKey }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(mockRes.json).toHaveBeenCalledWith({ + error: "Metric not found", + }); + expect(executeMock).not.toHaveBeenCalled(); + }, + ); + + test("malformed registry → 503 METRIC_REGISTRY_LOAD_FAILED", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + // A real malformed config on disk (invalid JSON) → the loader throws → + // the route surfaces a 503, exercising the actual load path. + writeRegistry(dir, "{ not valid json"); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(503); + expect(mockRes.json).toHaveBeenCalledWith({ + error: "Metric registry not available", + code: "METRIC_REGISTRY_LOAD_FAILED", + }); + }); + + test("self-heal: a fixed config is picked up on the next request (no restart)", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + writeRegistry(dir, "{ not valid json"); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + // First request: malformed → 503, and the failure is NOT cached. + const res1 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }), + res1, + ); + expect(res1.status).toHaveBeenCalledWith(503); + + // Fix the file. mtime changes, so the next request re-parses and serves + // it — no server restart, no latched 503. + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + const res2 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }), + res2, + ); + expect(res2.status).not.toHaveBeenCalledWith(503); + expect(res2.status).not.toHaveBeenCalledWith(404); + expect(executeMock).toHaveBeenCalled(); + }); + + test("hot-reload: a new key added to a working config is visible next request", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "mv-route-")); + tempRegistryDirs.push(dir); + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }); + const plugin = new AnalyticsPlugin({ ...config, queriesDir: dir }); + const { router, getHandler } = createMockRouter(); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + + // `orders` is not registered yet → 404. + const res1 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "orders" }, + body: { measures: ["cnt"] }, + }), + res1, + ); + expect(res1.status).toHaveBeenCalledWith(404); + + // Add `orders` to the config. The mtime bump invalidates the cache, so + // the next request sees it without a restart. + writeRegistry(dir, { + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + orders: { key: "orders", source: "cat.sch.order_metrics", lane: "sp" }, + }); + const res2 = createMockResponse(); + await handler( + createMockRequest({ + params: { key: "orders" }, + body: { measures: ["cnt"] }, + }), + res2, + ); + expect(res2.status).not.toHaveBeenCalledWith(404); + expect(executeMock).toHaveBeenCalled(); + }); + + test("no metric-views.json present → registry empty, unknown key 404, nothing executes", async () => { + const plugin = new AnalyticsPlugin(config); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + // Registry lazily loads from cwd; no config/queries/metric-views.json in + // the test cwd → empty registry (dormant). + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(404); + expect(executeMock).not.toHaveBeenCalled(); + }); + }); +}); + +// ── loadMetricRegistry: config parse against the landed metricSourceSchema. +describe("loadMetricRegistry", () => { + let dir: string; + + beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), "mv-registry-")); + }); + + afterEach(() => { + __resetMetricRegistryCache(); + rmSync(dir, { recursive: true, force: true }); + }); + + test("absent metric-views.json → empty registry (dormancy)", async () => { + expect(await loadMetricRegistry(dir)).toEqual({}); + }); + + test("derives lane from executor (default sp, user → obo)", async () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { + revenue: { source: "cat.sch.revenue_metrics" }, + customers: { source: "cat.sch.customer_metrics", executor: "user" }, + }, + }), + ); + + const registry = await loadMetricRegistry(dir); + expect(registry.revenue).toEqual({ + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }); + expect(registry.customers).toEqual({ + key: "customers", + source: "cat.sch.customer_metrics", + lane: "obo", + }); + }); + + test("registry has a null prototype (no inherited-property lookups)", async () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + + const registry = await loadMetricRegistry(dir); + expect(Object.getPrototypeOf(registry)).toBeNull(); + // A key that would resolve to a truthy inherited member on a plain object + // resolves to undefined here. + expect((registry as Record).toString).toBeUndefined(); + expect((registry as Record).__proto__).toBeUndefined(); + }); + + test("absent file yields a null-prototype (dormant) registry too", async () => { + // The ENOENT dormancy path must also be prototype-free — a metric key + // colliding with an inherited member can't 200 against an empty registry. + const registry = await loadMetricRegistry(dir); + expect(Object.getPrototypeOf(registry)).toBeNull(); + }); + + test("malformed JSON throws", async () => { + writeFileSync(path.join(dir, "metric-views.json"), "{ not json"); + await expect(loadMetricRegistry(dir)).rejects.toThrow(/Failed to parse/); + }); + + test("schema-invalid config throws", async () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "not-a-three-part-fqn" } }, + }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("rejects more than 200 metric views (runtime parity with typegen cap)", async () => { + // A config that fails type generation (MAX_METRIC_VIEWS) must not silently + // pass at runtime. z.record has no `.max` in zod 4, so this is enforced by + // the schema's superRefine. + const metricViews: Record = {}; + for (let i = 0; i < 201; i++) { + metricViews[`m_${i}`] = { source: `cat.sch.view_${i}` }; + } + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("rejects an FQN segment longer than 255 chars (per-segment cap)", async () => { + // The per-segment length cap is not expressible as a whole-string + // maxLength; the schema's superRefine enforces it so runtime matches the + // typegen resolver. + const longSegment = "a".repeat(256); + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: `cat.sch.${longSegment}` } }, + }), + ); + await expect(loadMetricRegistry(dir)).rejects.toThrow( + /Invalid metric-views.json/, + ); + }); + + test("accepts exactly 200 views and a 255-char segment (boundary)", async () => { + const metricViews: Record = {}; + for (let i = 0; i < 200; i++) { + metricViews[`m_${i}`] = { source: `cat.sch.view_${i}` }; + } + // One entry with a segment at exactly the 255 limit — must pass. + metricViews.m_0 = { source: `cat.sch.${"a".repeat(255)}` }; + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ metricViews }), + ); + await expect(loadMetricRegistry(dir)).resolves.toBeDefined(); + }); + + test("re-reads only after the file changes (mtime-validated cache)", async () => { + // getMetricRegistry caches by dir and revalidates via stat; a second call + // with no change returns the same object, and a rewrite is picked up. + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + const first = await getMetricRegistry(dir); + const second = await getMetricRegistry(dir); + expect(second).toBe(first); // same cached object, no re-parse + + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { + revenue: { source: "cat.sch.revenue_metrics" }, + orders: { source: "cat.sch.order_metrics" }, + }, + }), + ); + const third = await getMetricRegistry(dir); + expect(third).not.toBe(first); + expect(Object.keys(third).sort()).toEqual(["orders", "revenue"]); + }); + + test("picks up a SAME-SIZE edit (ctime invalidation, not just size)", async () => { + // The failure mode size+mtime alone would miss: rewrite the config to the + // EXACT same byte length (repoint `source` to an equal-length FQN). `size` + // is unchanged and a coarse-mtime FS might not advance `mtimeMs`, but + // `ctimeMs` bumps on any write — so the new source must be served. + const p = path.join(dir, "metric-views.json"); + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_aaa" } }, + }), + ); + const before = await getMetricRegistry(dir); + expect(before.revenue?.source).toBe("cat.sch.rev_aaa"); + + // Same-length replacement: "rev_aaa" → "rev_bbb" keeps the file byte-count + // identical, so `size` cannot disambiguate. + const sizeBefore = statSync(p).size; + writeFileSync( + p, + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.rev_bbb" } }, + }), + ); + expect(statSync(p).size).toBe(sizeBefore); // same size, as designed + + const after = await getMetricRegistry(dir); + expect(after.revenue?.source).toBe("cat.sch.rev_bbb"); + }); + + test("__resetMetricRegistryCache forces a cold re-read", async () => { + writeFileSync( + path.join(dir, "metric-views.json"), + JSON.stringify({ + metricViews: { revenue: { source: "cat.sch.revenue_metrics" } }, + }), + ); + const first = await getMetricRegistry(dir); + __resetMetricRegistryCache(); + const second = await getMetricRegistry(dir); + // Cleared cache → fresh parse → a new object (not the memoized instance). + expect(second).not.toBe(first); + expect(Object.keys(second)).toEqual(["revenue"]); + }); +}); + +// ── Phase 2: the structured filter engine (translator + validator). +// Registry-free: names are grammar-gated, values are parameterized. No +// allowlist, no op⇄dimension-type check. +describe("metric — filter translator", () => { + const registration: MetricRegistration = { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }; + + // Render a filter via buildMetricSql and return the WHERE fragment + params. + function render(filter: unknown) { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + filter: filter as never, + }); + const match = statement.match(/ WHERE (.+?)( GROUP BY| LIMIT|$)/); + const where = match ? match[1] : null; + return { statement, where, parameters }; + } + + describe("operators (12 unit tests)", () => { + test("equals → ` = :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "equals", + values: ["EMEA"], + }); + expect(where).toBe("`region` = :f_0"); + expect(parameters).toEqual({ + f_0: { __sql_type: "STRING", value: "EMEA" }, + }); + }); + + test("notEquals → ` <> :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notEquals", + values: ["EMEA"], + }); + expect(where).toBe("`region` <> :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); + }); + + test("in → ` IN (:f_0, :f_1, ...)`", () => { + const { where, parameters } = render({ + member: "region", + operator: "in", + values: ["EMEA", "APAC", "AMER"], + }); + expect(where).toBe("`region` IN (:f_0, :f_1, :f_2)"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "EMEA" }); + expect(parameters.f_1).toEqual({ __sql_type: "STRING", value: "APAC" }); + expect(parameters.f_2).toEqual({ __sql_type: "STRING", value: "AMER" }); + }); + + test("notIn → ` NOT IN (:f_0, :f_1, ...)`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notIn", + values: ["EMEA", "APAC"], + }); + expect(where).toBe("`region` NOT IN (:f_0, :f_1)"); + expect(Object.keys(parameters)).toHaveLength(2); + }); + + test("gt → ` > :f_0` (numeric value bound as INT)", () => { + const { where, parameters } = render({ + member: "deal_size", + operator: "gt", + values: [10000], + }); + expect(where).toBe("`deal_size` > :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "INT", value: "10000" }); + }); + + test("gte → ` >= :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "gte", + values: [5000], + }); + expect(where).toBe("`deal_size` >= :f_0"); + }); + + test("lt → ` < :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "lt", + values: [100], + }); + expect(where).toBe("`deal_size` < :f_0"); + }); + + test("lte → ` <= :f_0`", () => { + const { where } = render({ + member: "deal_size", + operator: "lte", + values: [50000], + }); + expect(where).toBe("`deal_size` <= :f_0"); + }); + + test("contains → ` LIKE :f_0` (value wrapped in %...%)", () => { + const { where, parameters } = render({ + member: "region", + operator: "contains", + values: ["MEA"], + }); + expect(where).toBe("`region` LIKE :f_0"); + expect(parameters.f_0).toEqual({ __sql_type: "STRING", value: "%MEA%" }); + }); + + test("notContains → ` NOT LIKE :f_0`", () => { + const { where, parameters } = render({ + member: "region", + operator: "notContains", + values: ["test"], + }); + expect(where).toBe("`region` NOT LIKE :f_0"); + expect(parameters.f_0).toEqual({ + __sql_type: "STRING", + value: "%test%", + }); + }); + + test("set → ` IS NOT NULL` (no bind)", () => { + const { where, parameters } = render({ + member: "region", + operator: "set", + }); + expect(where).toBe("`region` IS NOT NULL"); + expect(parameters).toEqual({}); + }); + + test("notSet → ` IS NULL` (no bind)", () => { + const { where, parameters } = render({ + member: "region", + operator: "notSet", + }); + expect(where).toBe("`region` IS NULL"); + expect(parameters).toEqual({}); + }); + }); + + describe("AND/OR composition", () => { + test("flat AND group renders predicates joined by AND", () => { + const { where, parameters } = render({ + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Enterprise"] }, + ], + }); + expect(where).toBe("(`region` = :f_0 AND `segment` = :f_1)"); + expect(parameters.f_0.value).toBe("EMEA"); + expect(parameters.f_1.value).toBe("Enterprise"); + }); + + test("flat OR group renders predicates joined by OR", () => { + const { where } = render({ + or: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "region", operator: "equals", values: ["APAC"] }, + ], + }); + expect(where).toBe("(`region` = :f_0 OR `region` = :f_1)"); + }); + + test("AND-of-OR composes nested groups", () => { + const { where } = render({ + and: [ + { member: "region", operator: "in", values: ["EMEA", "APAC"] }, + { + or: [ + { member: "segment", operator: "equals", values: ["Enterprise"] }, + { member: "deal_size", operator: "gt", values: [50000] }, + ], + }, + ], + }); + expect(where).toContain("(`region` IN ("); + expect(where).toContain(" AND "); + expect(where).toContain("OR"); + }); + + test("OR-of-AND composes nested groups", () => { + const { where } = render({ + or: [ + { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Enterprise"] }, + ], + }, + { member: "region", operator: "equals", values: ["APAC"] }, + ], + }); + expect(where).toMatch(/^\(.+ OR .+\)$/); + expect(where).toContain(" AND "); + }); + + test("empty `and: []` group emits no WHERE clause", () => { + const { statement, parameters } = buildMetricSql(registration, { + measures: ["arr"], + filter: { and: [] }, + }); + expect(statement).not.toContain("WHERE"); + expect(parameters).toEqual({}); + }); + + test("empty `or: []` renders 1 = 0 (defense in depth past the validator)", () => { + // The validator rejects `or: []` (see cardinality tests), but if a bypass + // ever reaches the renderer, an empty disjunction must render vacuous- + // false — never drop the predicate (which would mean "match everything"). + const { where } = render({ or: [] }); + expect(where).toBe("1 = 0"); + }); + }); + + describe("depth cap", () => { + test("rejects 9 levels of AND nesting at preCheckFilterDepth (before Zod)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 9; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).toThrowError(/fields:.*filter/); + }); + + test("accepts exactly 8 levels of AND nesting (validator)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 8; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).not.toThrow(); + }); + + test("renderFilter independently rejects nesting past the depth cap", () => { + // Second, independent depth enforcement: even if a payload bypasses the + // pre-Zod pre-check and Zod's superRefine, the SQL renderer re-checks the + // depth as defense in depth. Build 9-deep and call buildMetricSql + // directly (skipping validateMetricRequest). + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 9; i += 1) { + node = { and: [node] }; + } + expect(() => + buildMetricSql(registration, { + measures: ["arr"], + filter: node as never, + }), + ).toThrow(/nesting exceeds the maximum depth/); + }); + + test("rejects pathologically deep filter without stack-overflow (pre-parse cap)", () => { + let node: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 10_000; i += 1) { + node = { and: [node] }; + } + expect(() => + validateMetricRequest({ measures: ["arr"], filter: node }), + ).toThrowError(/fields:.*filter/); + }); + + test("rejects deep `or` even when paired with empty `and` (else-if bypass guard)", () => { + let deepOr: unknown = { + member: "region", + operator: "equals", + values: ["EMEA"], + }; + for (let i = 0; i < 10_000; i += 1) { + deepOr = { or: [deepOr] }; + } + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { and: [], or: [deepOr] } as never, + }), + ).toThrowError(/fields:.*filter/); + }); + + test("rejects breadth-DoS: a single group with too many children", () => { + const wide = Array.from({ length: 1000 }, () => ({ + member: "region", + operator: "equals" as const, + values: ["EMEA"], + })); + expect(() => + validateMetricRequest({ measures: ["arr"], filter: { and: wide } }), + ).toThrowError(/fields:.*filter/); + }); + }); + + describe("parameterization safety (no values in rendered SQL)", () => { + test("string values do not appear verbatim in the SQL string", () => { + const sneaky = "EMEA' OR '1'='1"; + const { statement } = render({ + member: "region", + operator: "equals", + values: [sneaky], + }); + expect(statement).not.toContain(sneaky); + expect(statement).toContain(":f_0"); + }); + + test("numeric values do not appear verbatim in the SQL string", () => { + const { statement } = render({ + member: "deal_size", + operator: "gt", + values: [987654321], + }); + expect(statement).not.toContain("987654321"); + expect(statement).toContain(":f_0"); + }); + + test("LIKE wildcard is the only value transformation; the original string is not in SQL", () => { + const { statement } = render({ + member: "region", + operator: "contains", + values: ["dangerous%"], + }); + expect(statement).not.toContain("dangerous"); + expect(statement).toContain(":f_0"); + }); + + test("IN values are individually bound (not concatenated)", () => { + const { statement, parameters } = render({ + member: "region", + operator: "in", + values: ["A", "B", "C"], + }); + expect(statement).not.toMatch(/region IN \([^:]/); + expect(Object.keys(parameters)).toHaveLength(3); + }); + + test("an injection-shaped filter member is neutralized by quoting", () => { + // No control char → a valid (if nonexistent) column name: quoted whole, + // the `;`/`DROP`/`--` are inert inside the backtick-delimited identifier. + const { where } = render({ + member: "region; DROP TABLE foo --", + operator: "equals", + values: ["x"], + }); + expect(where).toBe("`region; DROP TABLE foo --` = :f_0"); + }); + + test("a filter member with a control character throws before SQL is built", () => { + expect(() => + render({ + member: "region\ndrop", + operator: "equals", + values: ["x"], + }), + ).toThrowError(/not a valid identifier|control character/); + }); + }); + + describe("validator — operator + cardinality (registry-free)", () => { + test("rejects an unknown operator", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + member: "region", + operator: "startsWith" as never, + values: ["E"], + }, + }), + ).toThrowError(/fields:.*filter\.operator/); + }); + + test("rejects equals with zero values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "equals", values: [] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects equals with multiple values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "equals", values: ["A", "B"] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects in with empty values (cardinality)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "in", values: [] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects set with values (cardinality — must be absent)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "set", values: ["EMEA"] }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("accepts set with no values", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "set" }, + }), + ).not.toThrow(); + }); + + test("accepts notSet with an empty values array", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "notSet", values: [] }, + }), + ).not.toThrow(); + }); + + test("rejects `contains` with a non-string value", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + member: "region", + operator: "contains", + values: [42 as unknown as string], + }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects a filter predicate with too many values (DoS guard)", () => { + const big = Array.from({ length: 2000 }, (_, i) => `v${i}`); + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "region", operator: "in", values: big }, + }), + ).toThrowError(/fields:.*filter\.values/); + }); + + test("rejects a bare-array filter (not a Predicate or { and }/{ or } group)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: [ + { member: "region", operator: "in", values: ["EMEA"] }, + ] as never, + }), + ).toThrow(); + }); + + test("rejects empty `or` group (empty disjunction is vacuously false)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { or: [] }, + }), + ).toThrowError(/fields:.*filter\.or/); + }); + + test("accepts empty `and` group (no constraint contributed)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { and: [] }, + }), + ).not.toThrow(); + }); + + test("rejects an unknown operator at depth (nested filter)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { + member: "segment", + operator: "matches" as never, + values: ["X"], + }, + ], + }, + }), + ).toThrowError(/fields:.*filter\.and\.1\.operator/); + }); + + test("well-formed-but-unknown member is NOT rejected locally (no allowlist)", () => { + // The dropped-allowlist posture: a grammar-valid member that isn't a + // real dimension passes validation AND SQL construction — it reaches the + // warehouse, which is the authority on whether the column exists. + expect(() => + validateMetricRequest({ + measures: ["arr"], + filter: { member: "ghost_column", operator: "equals", values: ["x"] }, + }), + ).not.toThrow(); + const { where } = render({ + member: "ghost_column", + operator: "equals", + values: ["x"], + }); + expect(where).toBe("`ghost_column` = :f_0"); + }); + }); + + describe("validator — measure/dimension uniqueness", () => { + // Measures and dimensions become SELECT columns aliased to their own name; + // a repeated name collapses to a single row-object key and silently drops + // a value during row materialization, so uniqueness is enforced up front. + test("rejects a duplicate measure", () => { + expect(() => + validateMetricRequest({ measures: ["arr", "arr"] }), + ).toThrowError(/fields:.*measures/); + }); + + test("rejects a duplicate dimension", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region", "region"], + }), + ).toThrowError(/fields:.*measures/); + }); + + test("rejects a name that is BOTH a measure and a dimension", () => { + // The corruption case: `SELECT MEASURE(`x`) AS `x`, x ... GROUP BY ALL` + // materializes two `x` columns and the second overwrites the first. + expect(() => + validateMetricRequest({ + measures: ["revenue"], + dimensions: ["revenue"], + }), + ).toThrowError(/fields:.*measures/); + }); + + test("accepts distinct measures and dimensions (no false positive)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr", "mrr"], + dimensions: ["region", "segment"], + }), + ).not.toThrow(); + }); + }); + + describe("validator — format (JSON-only at v1)", () => { + test("accepts JSON_ARRAY", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "JSON_ARRAY" }), + ).not.toThrow(); + }); + + test("accepts the legacy JSON alias (normalizes to JSON_ARRAY)", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "JSON" }), + ).not.toThrow(); + }); + + test("rejects ARROW_STREAM (not implemented on the metric route)", () => { + // Fail loud rather than silently deliver JSON for an Arrow request. + expect(() => + validateMetricRequest({ measures: ["arr"], format: "ARROW_STREAM" }), + ).toThrowError(/fields:.*format/); + }); + + test("rejects the legacy ARROW alias (normalizes to ARROW_STREAM)", () => { + expect(() => + validateMetricRequest({ measures: ["arr"], format: "ARROW" }), + ).toThrowError(/fields:.*format/); + }); + }); + + describe("timeGrain + timeDimension (validator)", () => { + test("a grammar-valid timeGrain + timeDimension (in dimensions) is accepted", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date", + }), + ).not.toThrow(); + }); + + test("a timeGrain failing TIME_GRAIN_PATTERN is rejected", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "MONTH; DROP TABLE t", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeGrain/); + }); + + test("a capitalized grain (Month) is rejected by the grammar gate", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "Month", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeGrain/); + }); + + test("a timeDimension failing DIMENSION_NAME_PATTERN is rejected", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date; DROP TABLE t", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeGrain without timeDimension → 400 (grain requires a target)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["order_date"], + timeGrain: "day", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeDimension not in dimensions → 400 (must be selectable + in GROUP BY ALL)", () => { + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region"], + timeGrain: "month", + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + + test("timeDimension not in dimensions → 400 even without timeGrain", () => { + // The dimensions-membership rule holds independent of timeGrain: a + // timeDimension that isn't selected can never be bucketed or grouped. + expect(() => + validateMetricRequest({ + measures: ["arr"], + dimensions: ["region"], + timeDimension: "order_date", + }), + ).toThrowError(/fields:.*timeDimension/); + }); + }); + + describe("sort-before-hash (predicate ordering inside groups)", () => { + test("predicate order does not affect the rendered SQL within an AND group", () => { + const a = render({ + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Ent"] }, + ], + }); + const b = render({ + and: [ + { member: "segment", operator: "equals", values: ["Ent"] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + }); + expect(a.where).toBe(b.where); + }); + }); + + // The warehouse-authoritative unknown-name parity test (sanitized + // clientMessage/errorCode envelope) lands here because the Phase 1 harness + // can drive the metric route end-to-end and assert on the SSE error bytes. + describe("warehouse-authoritative unknown-name parity", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + test("a well-formed-but-unknown measure reaches the warehouse and surfaces a sanitized error envelope", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + // The warehouse rejects the unknown column. The raw text carries the + // offending name; the connector wraps it in an ExecutionError whose + // `.message` is server-log-only and whose `clientMessage` is sanitized. + const { ExecutionError } = await import("../../../errors/execution"); + const rawWarehouseText = + "[UNRESOLVED_COLUMN] A column with name `ghost_measure` cannot be resolved"; + const executeMock = vi + .fn() + .mockRejectedValue( + ExecutionError.statementFailed(rawWarehouseText, "UNRESOLVED_COLUMN"), + ); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + // grammar-valid, but not a real measure — NOT rejected locally. + body: { measures: ["ghost_measure"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The well-formed-unknown measure was NOT rejected locally — it reached + // the warehouse (parity with the raw `.sql` flow). + expect(executeMock).toHaveBeenCalled(); + expect(executeMock.mock.calls[0][1]).toEqual( + expect.objectContaining({ + statement: + "SELECT MEASURE(`ghost_measure`) AS `ghost_measure` FROM `cat`.`sch`.`revenue_metrics`", + }), + ); + + // The SSE error envelope carries a sanitized clientMessage + structured + // errorCode — the raw warehouse text (with the offending column name) + // never reaches the client payload (#329 scrubbing, inherited). + const errorWrites = (mockRes.write as any).mock.calls + .map((call: any[]) => call[0] as string) + .filter((s: string) => s.startsWith("data: ")); + const errorPayload = errorWrites.find((s: string) => + s.includes('"errorCode"'), + ); + expect(errorPayload).toBeTruthy(); + expect(errorPayload).toContain('"errorCode":"UNRESOLVED_COLUMN"'); + expect(errorPayload).toContain('"error":"Query execution failed"'); + expect(errorPayload).not.toContain("ghost_measure"); + expect(errorPayload).not.toContain("UNRESOLVED_COLUMN]"); + }); + }); +}); + +// ── Phase 3: cache-key composition. `composeMetricCacheKey` produces the +// array `CacheManager.generateKey` concatenates + sha256s; the invariants +// below are what make the cache both correct (semantically equal calls collapse) +// and safe (distinct args / executors never collide). +describe("composeMetricCacheKey", () => { + const base: { + metricKey: string; + source: string; + measures: string[]; + format: string; + executorKey: string; + } = { + metricKey: "revenue", + source: "cat.sch.revenue_metrics", + measures: ["arr"], + format: "JSON_ARRAY", + executorKey: "sp", + }; + + test("same key but different source → different keys (config repoint is not stale-served)", () => { + const a = composeMetricCacheKey({ ...base, source: "cat.sch.old_view" }); + const b = composeMetricCacheKey({ ...base, source: "cat.sch.new_view" }); + expect(a).not.toEqual(b); + }); + + test("timeDimension does NOT fork the key when timeGrain is absent (no SQL effect)", () => { + // Without a grain, renderDimensionClause emits the bare column, so + // timeDimension has no effect on the SQL — two such calls must cache-hit. + const withTd = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeDimension: "order_date", + }); + const withoutTd = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + }); + expect(withTd).toEqual(withoutTd); + }); + + test("timeDimension DOES fork the key when timeGrain is set (changes the SQL)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "region"], + timeGrain: "month", + timeDimension: "region", + }); + expect(a).not.toEqual(b); + }); + + test("measure ORDER does not affect the key (sorted before hashing)", () => { + const a = composeMetricCacheKey({ + ...base, + measures: ["arr", "revenue"], + }); + const b = composeMetricCacheKey({ + ...base, + measures: ["revenue", "arr"], + }); + expect(a).toEqual(b); + }); + + test("dimension ORDER does not affect the key (sorted before hashing)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["region", "segment"], + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["segment", "region"], + }); + expect(a).toEqual(b); + }); + + test("different measures → different keys", () => { + const a = composeMetricCacheKey({ ...base, measures: ["arr"] }); + const b = composeMetricCacheKey({ ...base, measures: ["mrr"] }); + expect(a).not.toEqual(b); + }); + + test("different dimensions → different keys", () => { + const a = composeMetricCacheKey({ ...base, dimensions: ["region"] }); + const b = composeMetricCacheKey({ ...base, dimensions: ["segment"] }); + expect(a).not.toEqual(b); + }); + + test("different timeGrain → different keys", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date"], + timeGrain: "month", + timeDimension: "order_date", + }); + expect(a).not.toEqual(b); + }); + + test("different timeDimension → different keys (new field salts the key)", () => { + const a = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "ship_date"], + timeGrain: "day", + timeDimension: "order_date", + }); + const b = composeMetricCacheKey({ + ...base, + dimensions: ["order_date", "ship_date"], + timeGrain: "day", + timeDimension: "ship_date", + }); + expect(a).not.toEqual(b); + }); + + test("different limit → different keys", () => { + const a = composeMetricCacheKey({ ...base, limit: 10 }); + const b = composeMetricCacheKey({ ...base, limit: 20 }); + expect(a).not.toEqual(b); + }); + + test("predicate ORDER inside a group does not affect the key (canonicalizeFilter)", () => { + const a = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "region", operator: "equals", values: ["EMEA"] }, + { member: "segment", operator: "equals", values: ["Ent"] }, + ], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "segment", operator: "equals", values: ["Ent"] }, + { member: "region", operator: "equals", values: ["EMEA"] }, + ], + } as MetricFilter, + }); + expect(a).toEqual(b); + }); + + test("predicate ORDER is stable for two ops on the SAME member (sort-key delimiter)", () => { + // Same member, different operators — the sort key is `${member}/${operator}`, + // so the "/" delimiter (not a raw concatenation) is what disambiguates the + // pair and keeps the ordering — and thus the key — independent of input + // order. Two range bounds on one column is the everyday shape of this. + const a = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "revenue", operator: "gt", values: [100] }, + { member: "revenue", operator: "lt", values: [200] }, + ], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + and: [ + { member: "revenue", operator: "lt", values: [200] }, + { member: "revenue", operator: "gt", values: [100] }, + ], + } as MetricFilter, + }); + expect(a).toEqual(b); + }); + + test("distinct filter VALUES → distinct keys", () => { + const a = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "equals", + values: ["EMEA"], + } as MetricFilter, + }); + const b = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "equals", + values: ["APAC"], + } as MetricFilter, + }); + expect(a).not.toEqual(b); + }); + + test("a filter present vs absent → different keys", () => { + const withFilter = composeMetricCacheKey({ + ...base, + filter: { + member: "region", + operator: "set", + } as MetricFilter, + }); + const withoutFilter = composeMetricCacheKey({ ...base }); + expect(withFilter).not.toEqual(withoutFilter); + }); + + test("SP vs OBO (same args) → different keys via executorKey", () => { + const sp = composeMetricCacheKey({ ...base, executorKey: "sp" }); + const obo = composeMetricCacheKey({ + ...base, + executorKey: deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }), + }); + expect(sp).not.toEqual(obo); + }); +}); + +// ── Phase 3: executor-key isolation. The key is what scopes the cache — `"sp"` +// shares it across all callers, a per-user hash isolates OBO callers. The raw +// identity must never enter the key verbatim (privacy: cache keys are logged +// and persisted). +describe("deriveMetricExecutorKey", () => { + test("SP lane → literal 'sp' (shared cache)", () => { + expect(deriveMetricExecutorKey({ lane: "sp" })).toBe("sp"); + }); + + test("SP lane ignores any supplied identity", () => { + expect(deriveMetricExecutorKey({ lane: "sp", userIdentity: "alice" })).toBe( + "sp", + ); + }); + + test("OBO lane hashes the identity — raw identity never appears in the key", () => { + const key = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice@example.com", + }); + expect(key).toMatch(/^[a-f0-9]{64}$/); + expect(key).not.toContain("alice"); + }); + + test("OBO same identity → stable key (cache hits work)", () => { + const a = deriveMetricExecutorKey({ lane: "obo", userIdentity: "alice" }); + const b = deriveMetricExecutorKey({ lane: "obo", userIdentity: "alice" }); + expect(a).toBe(b); + }); + + test("OBO identity is trimmed before hashing (whitespace does not fork the scope)", () => { + const padded = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: " alice ", + }); + const bare = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }); + expect(padded).toBe(bare); + }); + + test("OBO different identities → different keys (isolation holds)", () => { + const alice = deriveMetricExecutorKey({ + lane: "obo", + userIdentity: "alice", + }); + const bob = deriveMetricExecutorKey({ lane: "obo", userIdentity: "bob" }); + expect(alice).not.toBe(bob); + }); + + test("OBO empty identity → throws AuthenticationError (no shared 'anonymous' scope)", () => { + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: "" }), + ).toThrow(AuthenticationError); + }); + + test("OBO whitespace-only identity → throws AuthenticationError", () => { + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: " " }), + ).toThrow(AuthenticationError); + }); + + test("OBO missing (undefined/null) identity → throws AuthenticationError", () => { + expect(() => deriveMetricExecutorKey({ lane: "obo" })).toThrow( + AuthenticationError, + ); + expect(() => + deriveMetricExecutorKey({ lane: "obo", userIdentity: null }), + ).toThrow(AuthenticationError); + }); +}); + +// ── Phase 3: lane dispatch at the handler level. The lane comes from the +// registration (the entry's `executor` in metric-views.json), NOT the URL: +// OBO-lane routes through `asUser(req)`, SP-lane through the default executor. +// A missing/whitespace OBO identity must land on the canonical 401 envelope, +// never an out-of-envelope 500. +describe("metric route — lane dispatch (Phase 3)", () => { + let config: IAnalyticsConfig; + let serviceContextMock: Awaited>; + + beforeEach(async () => { + config = { timeout: 5000 }; + setupDatabricksEnv(); + mockCacheStore.clear(); + ServiceContext.reset(); + serviceContextMock = await mockServiceContext(); + }); + + afterEach(() => { + serviceContextMock?.restore(); + }); + + test("OBO-lane registration routes through asUser(req)", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + const asUserSpy = vi.spyOn(plugin, "asUser"); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": "alice", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + // The OBO lane dispatched through the user-scoped executor… + expect(asUserSpy).toHaveBeenCalledWith(mockReq); + // …and the SQL still reached the warehouse and streamed a result. + expect(executeMock).toHaveBeenCalled(); + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + }); + + test("SP-lane registration uses the default executor (asUser not called)", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "sp", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + const asUserSpy = vi.spyOn(plugin, "asUser"); + const executeMock = vi + .fn() + .mockResolvedValue({ result: { data: [{ arr: 1 }] } }); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + // Even with user headers present, an SP-lane metric must NOT impersonate. + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": "alice", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(asUserSpy).not.toHaveBeenCalled(); + expect(executeMock).toHaveBeenCalled(); + expect(mockRes.write).toHaveBeenCalledWith("event: result\n"); + }); + + test("OBO metric with no user identity → canonical 401, no SQL executed", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + // No x-forwarded-* headers at all → the OBO dispatch throws an + // AuthenticationError, caught and returned as a canonical 401 envelope. + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(401); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ code: "AUTHENTICATION_ERROR" }), + ); + // Landed on the 401 path before any SQL and without opening the SSE stream. + expect(executeMock).not.toHaveBeenCalled(); + expect(mockRes.write).not.toHaveBeenCalled(); + }); + + test("OBO metric with whitespace-only user identity → canonical 401", async () => { + const plugin = new AnalyticsPlugin({ + ...config, + queriesDir: registryDir({ + revenue: { + key: "revenue", + source: "cat.sch.revenue_metrics", + lane: "obo", + }, + }), + }); + const { router, getHandler } = createMockRouter(); + + const executeMock = vi.fn(); + (plugin as any).SQLClient.executeStatement = executeMock; + + plugin.injectRoutes(router); + const handler = getHandler("POST", "/metric/:key"); + const mockReq = createMockRequest({ + params: { key: "revenue" }, + body: { measures: ["arr"] }, + headers: { + "x-forwarded-access-token": "user-token", + "x-forwarded-user": " ", + }, + }); + const mockRes = createMockResponse(); + + await handler(mockReq, mockRes); + + expect(mockRes.status).toHaveBeenCalledWith(401); + expect(mockRes.json).toHaveBeenCalledWith( + expect.objectContaining({ code: "AUTHENTICATION_ERROR" }), + ); + expect(executeMock).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/appkit/src/plugins/analytics/types.ts b/packages/appkit/src/plugins/analytics/types.ts index 95c987d1..c01c04e2 100644 --- a/packages/appkit/src/plugins/analytics/types.ts +++ b/packages/appkit/src/plugins/analytics/types.ts @@ -23,6 +23,17 @@ export interface IAnalyticsConfig extends BasePluginConfig { * byte arrives the stream is not time-bounded. */ arrowFirstByteTimeoutMs?: number; + /** + * Directory to read the metric-view registry (`metric-views.json`) from. + * + * @internal FOR TESTS ONLY. Production always reads the fixed + * `config/queries/` path (the default), the same directory the `.sql` query + * path uses — do NOT set this to relocate config in a real app. It exists so + * tests can point the loader at a temp directory and exercise the real + * stat → read → cache path against a real file, rather than poking private + * plugin state or stubbing the loader. + */ + queriesDir?: string; } /** @@ -112,3 +123,99 @@ export interface AnalyticsQueryResponse { row_count: number; data: any[]; } + +// ──────────────────────────────────────────────────────────────────────────── +// Metric views — POST /api/analytics/metric/:key +// ──────────────────────────────────────────────────────────────────────────── + +/** + * Execution lane for a registered metric view, derived from the entry's + * `executor` in `metric-views.json`: + * - `"sp"` ← `executor: "app_service_principal"` — queried as the app + * service principal (cache shared across all users). + * - `"obo"` ← `executor: "user"` — queried on-behalf-of the requesting + * user (per-user cache). OBO dispatch is wired in a later phase. + */ +export type MetricLane = "sp" | "obo"; + +/** + * A single registered metric view, loaded from `config/queries/metric-views.json`. + * + * The registration carries only what the runtime needs to build and dispatch + * SQL: the metric `key`, the three-part UC FQN `source`, and the `lane`. There + * is intentionally NO build-time measure/dimension metadata here — the security + * boundary is the grammar gate plus parameterized values, not a name allowlist, + * so the runtime never enumerates known measures/dimensions. + */ +export interface MetricRegistration { + key: string; + source: string; + lane: MetricLane; +} + +/** + * v1 filter operator vocabulary — exactly twelve names. The runtime tuple + * `METRIC_FILTER_OPERATORS` (next to the validator in `metric.ts`) is the + * server-side source of truth; this union mirrors it statically. + */ +export type MetricFilterOperatorName = + | "equals" + | "notEquals" + | "in" + | "notIn" + | "gt" + | "gte" + | "lt" + | "lte" + | "contains" + | "notContains" + | "set" + | "notSet"; + +/** + * A single filter predicate — the leaf node of the recursive + * {@link MetricFilter} tree. `member` is a dimension name (grammar-gated, not + * allowlisted); `values` is bound through parameterized `:f_` bind vars + * and never interpolated into the SQL string. + */ +export interface MetricPredicate { + member: string; + operator: MetricFilterOperatorName; + values?: ReadonlyArray; +} + +/** + * Recursive filter expression for the metric-view request body: a leaf + * {@link MetricPredicate} or an `{ and: [...] }` / `{ or: [...] }` group. The + * shape is intentionally non-generic server-side — per-metric narrowing (if + * any) lives client-side. + */ +export type MetricFilter = + | MetricPredicate + | { and: ReadonlyArray } + | { or: ReadonlyArray }; + +/** + * Validated request body for `POST /api/analytics/metric/:key`. + * + * `measures` is required. `dimensions` drive `GROUP BY ALL`; `filter` is the + * recursive structured predicate tree translated into a parameterized `WHERE` + * clause. `timeGrain` buckets the single dimension named by `timeDimension` + * via `date_trunc`; it requires `timeDimension`, and `timeDimension` must be + * one of `dimensions` so it is selected and in `GROUP BY ALL`. Both tokens are + * grammar-gated before they reach SQL. + */ +export interface IAnalyticsMetricRequest { + measures: string[]; + dimensions?: string[]; + filter?: MetricFilter; + timeGrain?: string; + /** + * The single dimension that `timeGrain` buckets via `date_trunc`. Must be + * one of `dimensions` (so it is selected and in `GROUP BY ALL`) and is + * required whenever `timeGrain` is set. Grammar-gated as a SQL identifier. + */ + timeDimension?: string; + limit?: number; + format?: AnalyticsFormat; +} diff --git a/packages/appkit/src/type-generator/mv-registry/config.ts b/packages/appkit/src/type-generator/mv-registry/config.ts index 1995f072..e91e3b95 100644 --- a/packages/appkit/src/type-generator/mv-registry/config.ts +++ b/packages/appkit/src/type-generator/mv-registry/config.ts @@ -94,26 +94,13 @@ function isValidMetricKey(key: string): boolean { return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key); } -/** - * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? - * - * Well-formed = exactly three non-empty, dot-separated segments, each a valid - * Unity Catalog object name per the shared {@link UC_FQN_PATTERN} (the single - * source of truth, also used by the canonical Zod schema). Used as the - * defense-in-depth re-check at the describe fetcher seam; {@link resolveMetricConfig} - * runs the same checks but with specific, staged error messages. - * - * @note Segment length ({@link MAX_FQN_SEGMENT_LENGTH}) is NOT checked here — - * an over-long but otherwise legal name is still "valid shape". The length cap - * is a separate concern enforced (with its own message) in resolveMetricConfig. - */ -export function isValidFqn(fqn: string): boolean { - const segments = fqn.split("."); - if (segments.length !== FQN_SEGMENT_COUNT) { - return false; - } - return segments.every((segment) => UC_FQN_PATTERN.test(segment)); -} +// `isValidFqn` (the three-part UC FQN predicate) now lives in the shared +// zod-free leaf alongside `UC_FQN_PATTERN` and `quoteFqnForSql`, so the +// type-generator and the analytics runtime share one grammar + one escaper. +// `resolveMetricConfig` below intentionally does NOT call it — it re-derives +// the same checks inline so it can emit specific, staged error messages +// (arity, per-segment charset, per-segment length) rather than a single +// boolean. /** * Field allowlists enforced by {@link resolveMetricConfig}. diff --git a/packages/appkit/src/type-generator/mv-registry/describe.ts b/packages/appkit/src/type-generator/mv-registry/describe.ts index db3231d6..a77caf75 100644 --- a/packages/appkit/src/type-generator/mv-registry/describe.ts +++ b/packages/appkit/src/type-generator/mv-registry/describe.ts @@ -1,7 +1,13 @@ import type { WorkspaceClient } from "@databricks/sdk-experimental"; +// Grammar + SQL-quoting for metric-view FQNs live together in the shared, +// zod-free leaf so the type-generator and the analytics runtime validate and +// escape against one source of truth (see the module doc in metric-fqn.ts). +import { + isValidFqn, + quoteFqnForSql, +} from "../../../../shared/src/schemas/metric-fqn"; import { type DescribeFormatMemo, describeAdaptive } from "../statement-result"; import type { DatabricksStatementExecutionResponse } from "../types"; -import { isValidFqn } from "./config"; import type { DescribeFetcher, MetricColumnMetadata } from "./types"; /** @@ -284,49 +290,6 @@ function inferTimeGrains(type: string): string[] | undefined { return undefined; } -/** - * Quote a dot-separated FQN for safe interpolation into a Spark/Databricks SQL - * statement. - * - * Each dot-split segment is wrapped in backtick-quoted-identifier syntax. The - * one character that can break out of a backtick-quoted identifier is the - * backtick itself, escaped by doubling (`` ` `` → `` `` ``) — so every backtick - * inside a segment is doubled before the segment is wrapped. Control characters - * and newlines have no valid escape inside a quoted identifier, so a segment - * containing one is rejected outright. - * - * This is a pure, standalone escaper: it is intentionally independent of FQN - * naming validation ({@link isValidFqn}). Naming validation decides whether an - * FQN is an acceptable metric source; this function only guarantees that - * whatever it is handed cannot break out of the quoted identifier it produces. - * - * An ordinary identifier is unchanged apart from the wrapping backticks: - * `catalog.schema.view` → `` `catalog`.`schema`.`view` ``. - * - * @param fqn - Dot-separated identifier (e.g. `catalog.schema.view`). - * @returns The backtick-quoted, escaped identifier ready for interpolation. - * @throws If any segment contains a control character or newline. - */ -export function quoteFqnForSql(fqn: string): string { - // Reject anything that cannot be represented inside a backtick-quoted - // identifier. \p{Cc} is the Unicode "control" category, which covers C0 - // (incl. \n, \r, \t), DEL, and C1 — i.e. every control character/newline. - const CONTROL_OR_NEWLINE = /\p{Cc}/u; - return fqn - .split(".") - .map((segment) => { - if (CONTROL_OR_NEWLINE.test(segment)) { - throw new Error( - `Cannot quote FQN segment "${segment}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, - ); - } - // Double every backtick — the only break-out from a backtick-quoted - // identifier — then wrap the whole segment in backticks. - return `\`${segment.replace(/`/g, "``")}\``; - }) - .join("."); -} - /** * Build a DescribeFetcher from a real WorkspaceClient + warehouseId. */ diff --git a/packages/appkit/src/type-generator/tests/mv-registry.test.ts b/packages/appkit/src/type-generator/tests/mv-registry.test.ts index 06886140..8a06aca6 100644 --- a/packages/appkit/src/type-generator/tests/mv-registry.test.ts +++ b/packages/appkit/src/type-generator/tests/mv-registry.test.ts @@ -3,13 +3,16 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; +// `quoteFqnForSql` now lives in the shared zod-free leaf alongside the FQN +// grammar (moved so the analytics runtime can reuse it); the describe seam +// imports it from there. +import { quoteFqnForSql } from "../../../../shared/src/schemas/metric-fqn"; import { metricSourceSchema } from "../../../../shared/src/schemas/metric-source"; import { readMetricConfig, resolveMetricConfig } from "../mv-registry/config"; import { createWorkspaceDescribeFetcher, extractMetricColumns, parseDescribeTableExtendedJson, - quoteFqnForSql, } from "../mv-registry/describe"; import { generateMetricTypeDeclarations } from "../mv-registry/render-types"; import { syncMetrics } from "../mv-registry/sync"; diff --git a/packages/shared/src/schemas/metric-fqn.ts b/packages/shared/src/schemas/metric-fqn.ts index b958cd1a..be96c162 100644 --- a/packages/shared/src/schemas/metric-fqn.ts +++ b/packages/shared/src/schemas/metric-fqn.ts @@ -81,3 +81,113 @@ export const MAX_UC_OBJECT_NAME_LENGTH = 255; */ // biome-ignore lint/suspicious/noControlCharactersInRegex: UC explicitly prohibits ASCII control characters in object names; this negated class encodes that rule. export const UC_FQN_PATTERN = /^[^\x00-\x20\x7f./]+$/; + +/** A metric view FQN is exactly three segments: catalog.schema.metric_view. */ +const FQN_SEGMENT_COUNT = 3; + +/** + * Total predicate: is `fqn` a well-formed three-part UC metric view FQN? + * + * Well-formed = exactly three non-empty, dot-separated segments, each a valid + * Unity Catalog object name per {@link UC_FQN_PATTERN}. This is the shared, + * zod-free grammar check reused by every layer that must agree on FQN shape: + * the type-generator's config resolver and describe seam, and the analytics + * runtime's SQL builder. It is the boolean sibling of the composed + * `UC_THREE_PART_FQN_PATTERN` regex in `./metric-source.ts` (which stays a + * regex so zod can emit a JSON-schema `pattern`); both derive their per-segment + * charset from {@link UC_FQN_PATTERN}, so they cannot diverge. + * + * @note Segment length ({@link MAX_UC_OBJECT_NAME_LENGTH}) is NOT checked here — + * an over-long but otherwise legal name is still "valid shape". Callers that + * care about the length cap enforce it separately with their own message. + * + * @example + * isValidFqn("main.analytics.revenue"); // true + * isValidFqn("prod-data.analytics.rev"); // true (hyphens are UC-legal) + * isValidFqn("main.analytics"); // false (only two segments) + */ +export function isValidFqn(fqn: string): boolean { + const segments = fqn.split("."); + if (segments.length !== FQN_SEGMENT_COUNT) { + return false; + } + return segments.every((segment) => UC_FQN_PATTERN.test(segment)); +} + +/** + * Quote a dot-separated FQN for safe interpolation into a Spark/Databricks SQL + * statement. + * + * Each dot-split segment is wrapped in backtick-quoted-identifier syntax. The + * one character that can break out of a backtick-quoted identifier is the + * backtick itself, escaped by doubling (`` ` `` → `` `` ``) — so every backtick + * inside a segment is doubled before the segment is wrapped. Control characters + * and newlines have no valid escape inside a quoted identifier, so a segment + * containing one is rejected outright. + * + * This is a pure, standalone escaper: it is intentionally independent of FQN + * naming validation ({@link isValidFqn}). Naming validation decides whether an + * FQN is an acceptable metric source; this function only guarantees that + * whatever it is handed cannot break out of the quoted identifier it produces. + * Grammar and quoting live together here so a metric source is validated and + * escaped against one shared source of truth. + * + * An ordinary identifier is unchanged apart from the wrapping backticks: + * `catalog.schema.view` → `` `catalog`.`schema`.`view` ``. + * + * @param fqn - Dot-separated identifier (e.g. `catalog.schema.view`). + * @returns The backtick-quoted, escaped identifier ready for interpolation. + * @throws If any segment contains a control character or newline. + */ +export function quoteFqnForSql(fqn: string): string { + return fqn.split(".").map(quoteIdentifier).join("."); +} + +/** + * The Unicode "control" category (`\p{Cc}`): C0 (incl. `\n`, `\r`, `\t`), DEL, + * and C1 — every control character/newline. These have no valid escape inside + * a backtick-quoted identifier, so a name containing one cannot be safely + * quoted and is rejected. + */ +const CONTROL_OR_NEWLINE = /\p{Cc}/u; + +/** + * Is `name` a column/measure/dimension identifier that {@link quoteIdentifier} + * can safely escape? True for any non-empty string free of control characters + * and newlines. + * + * This is the **column-identifier** grammar — deliberately broader than the + * FQN-segment grammar ({@link UC_FQN_PATTERN}, which also forbids `.` and `/` + * because they are FQN structural characters). A metric view's measure or + * dimension is a single *delimited* column identifier: once backtick-quoted it + * may legally contain dots, slashes, spaces, hyphens, and non-ASCII — anything + * but a control character. The type-generator emits DESCRIBE column names + * verbatim into the generated `measureKeys`/`dimensionKeys` unions, so the + * runtime must accept exactly what can be safely quoted, or a generated name + * would typecheck but fail at runtime. + */ +export function isValidColumnName(name: string): boolean { + return name.length > 0 && !CONTROL_OR_NEWLINE.test(name); +} + +/** + * Quote a SINGLE identifier (one column/measure/dimension name, or one FQN + * segment) as a backtick-delimited identifier for safe SQL interpolation. + * + * Unlike {@link quoteFqnForSql}, this does NOT split on `.` — the whole input + * is one identifier, so a column literally named `net.revenue` becomes + * `` `net.revenue` `` (one identifier), not `` `net`.`revenue` `` (two). The + * backtick — the only break-out character — is doubled; control characters and + * newlines have no valid escape and are rejected. + * + * @throws If `name` contains a control character or newline. + */ +export function quoteIdentifier(name: string): string { + if (CONTROL_OR_NEWLINE.test(name)) { + throw new Error( + `Cannot quote identifier "${name}" for SQL: it contains a control character or newline, which has no valid escape inside a backtick-quoted identifier.`, + ); + } + // Double every backtick, then wrap in backticks. + return `\`${name.replace(/`/g, "``")}\``; +} diff --git a/packages/shared/src/schemas/metric-source.ts b/packages/shared/src/schemas/metric-source.ts index 2e4e1dfd..4d64458a 100644 --- a/packages/shared/src/schemas/metric-source.ts +++ b/packages/shared/src/schemas/metric-source.ts @@ -18,7 +18,25 @@ */ import { z } from "zod"; -import { UC_FQN_PATTERN } from "./metric-fqn"; +import { MAX_UC_OBJECT_NAME_LENGTH, UC_FQN_PATTERN } from "./metric-fqn"; + +/** + * Safety cap on the number of declared metric views — a typo / DoS guard, NOT + * a Unity Catalog limit. Mirrors the type-generator's `MAX_METRIC_VIEWS` so + * runtime config validation and type generation accept exactly the same + * configs (a config that fails generation must not silently pass at runtime). + */ +const MAX_METRIC_VIEWS = 200; + +/** + * Whole-FQN length cap: three max-length UC object names plus the two dots. + * The per-segment cap ({@link MAX_UC_OBJECT_NAME_LENGTH}) is enforced in the + * `superRefine` below; this whole-string bound is the declarative half (it + * serializes to a JSON-schema `maxLength`, whereas the per-segment check — like + * the entry-count cap — cannot be expressed declaratively and lives in the + * refinement, so it is a runtime/type-generator gate only). + */ +const MAX_FQN_LENGTH = MAX_UC_OBJECT_NAME_LENGTH * 3 + 2; /** * Three-part Unity Catalog FQN matcher, composed from the single-segment @@ -62,6 +80,7 @@ export const metricEntrySchema = z source: z .string() .regex(UC_THREE_PART_FQN_PATTERN) + .max(MAX_FQN_LENGTH) .describe( "Three-part Unity Catalog FQN of the metric view: ..", ) @@ -94,7 +113,39 @@ export const metricSourceSchema = z .strict() .describe( "Schema for AppKit metric-views.json — declares Unity Catalog Metric View sources for the analytics plugin's metric-view path. Each entry under 'metricViews' binds a metric key to a UC metric view FQN and an executor ('app_service_principal' shared cache, or 'user' per-user cache). Object form (rather than bare string) at v1 enables future per-entry option growth without breaking changes.", - ); + ) + // Caps that cannot be expressed declaratively (zod 4's `z.record` has no + // `.max`, and a per-dot-segment length bound isn't a whole-string + // `maxLength`). Enforced here so runtime config validation matches the + // type-generator's `resolveMetricConfig` exactly — a config that fails type + // generation must not silently pass at runtime. These refinements are + // invisible to `z.toJSONSchema`, so the generated JSON schema carries only + // the declarative `maxLength` on `source`; runtime + type-generator remain + // the authoritative gates for the entry-count and per-segment caps. + .superRefine((value, ctx) => { + const entries = value.metricViews ? Object.entries(value.metricViews) : []; + + if (entries.length > MAX_METRIC_VIEWS) { + ctx.addIssue({ + code: "custom", + message: `too many metric views: ${entries.length} declared, exceeding the maximum of ${MAX_METRIC_VIEWS}`, + path: ["metricViews"], + }); + } + + for (const [key, entry] of entries) { + const segments = entry.source.split("."); + for (let i = 0; i < segments.length; i++) { + if (segments[i].length > MAX_UC_OBJECT_NAME_LENGTH) { + ctx.addIssue({ + code: "custom", + message: `metric source segment ${i + 1} is ${segments[i].length} characters, exceeding the per-segment maximum of ${MAX_UC_OBJECT_NAME_LENGTH}`, + path: ["metricViews", key, "source"], + }); + } + } + } + }); export type MetricKey = z.infer; export type MetricExecutor = z.infer;