plugin.ts

// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.

import { readFile } from 'node:fs/promises';
import { createHash } from 'node:crypto';
import { Plugin, PluginContext } from '@objectstack/core';
import { NodeMetadataManager } from './node-metadata-manager.js';
import { MemoryLoader } from './loaders/memory-loader.js';
import { DEFAULT_METADATA_TYPE_REGISTRY } from '@objectstack/spec/kernel';
import type { MetadataPluginConfig } from '@objectstack/spec/kernel';
import {
    SysMetadataObject,
    SysMetadataHistoryObject,
} from '@objectstack/platform-objects/metadata';

// `SysMetadataObject` + `SysMetadataHistoryObject` are the customer overlay
// storage substrate (ADR-0005). They must always be auto-provisioned so
// `PUT /api/v1/meta/{view,dashboard}/...` has a place to write. All other
// metadata types (object/view/flow/agent/tool/dashboard/app/...) live as
// JSON inside `sys_metadata` — there are no separate per-type tables. The
// previously shipped `SysObject` / `SysView` / `SysFlow` / `SysAgent` /
// `SysTool` projection objects were removed in 2026-05 (see ADR 0005
// addendum); the projection pipeline was removed at the same time.
const queryableMetadataObjects = [
    SysMetadataObject,
    SysMetadataHistoryObject,
];

// Map from ObjectStackDefinition field name to MetadataType name
const ARTIFACT_FIELD_TO_TYPE: Record<string, string> = {
    objects: 'object',
    objectExtensions: 'object_extension',
    apps: 'app',
    views: 'view',
    pages: 'page',
    dashboards: 'dashboard',
    reports: 'report',
    actions: 'action',
    themes: 'theme',
    workflows: 'workflow',
    approvals: 'approval',
    flows: 'flow',
    roles: 'role',
    permissions: 'permission',
    sharingRules: 'sharing_rule',
    policies: 'policy',
    apis: 'api',
    webhooks: 'webhook',
    agents: 'agent',
    skills: 'skill',
    ragPipelines: 'rag_pipeline',
    hooks: 'hook',
    mappings: 'mapping',
    analyticsCubes: 'analytics_cube',
    connectors: 'connector',
    data: 'dataset',
};

export interface MetadataPluginOptions {
    rootDir?: string;
    watch?: boolean;
    config?: Partial<MetadataPluginConfig>;
    /** Organization ID for metadata-scoped consumers; MetadataPlugin itself does not persist runtime metadata. */
    organizationId?: string;
    /** Project ID used by local artifact envelopes and metadata-scoped consumers. */
    projectId?: string;
    /**
     * When set, MetadataPlugin loads metadata from an artifact instead of scanning
     * the filesystem. Only `local-file` is implemented now; `artifact-api` is
     * reserved for M3/M4.
     */
    artifactSource?:
        | { mode: 'local-file'; path: string; fetchTimeoutMs?: number }
        | { mode: 'artifact-api'; url: string; token?: string; commitId?: string; fetchTimeoutMs?: number };
    /**
     * Register the `sys_metadata` + `sys_metadata_history` storage objects
     * on this kernel. Default `true` for backward compatibility.
     *
     * Set to `false` for **per-project** kernels: in cloud / project mode the
     * control plane is the sole owner of metadata storage tables — exposing
     * them inside each project kernel would leak control-plane schema into
     * business-data namespaces.
     */
    registerSystemObjects?: boolean;
}

export class MetadataPlugin implements Plugin {
    name = 'com.objectstack.metadata';
    type = 'standard';
    version = '1.0.0';

    private manager: NodeMetadataManager;
    private options: MetadataPluginOptions;

    constructor(options: MetadataPluginOptions = {}) {
        this.options = {
            watch: true,
            ...options
        };

        const rootDir = this.options.rootDir || process.cwd();

        // Sealed-runtime carve-out: `bootstrap: 'artifact-only'` MUST NOT touch
        // the filesystem at all — that includes chokidar subscriptions. Force
        // watch off in that mode regardless of `options.watch`. The other two
        // modes ('eager', 'lazy') honor the user's flag; `lazy` + watch is a
        // valid combination because chokidar attaches to `rootDir` directly,
        // not as a side effect of any priming pass.
        const bootstrapMode = this.options.config?.bootstrap ?? 'eager';
        const effectiveWatch =
            bootstrapMode === 'artifact-only' ? false : (this.options.watch ?? true);

        this.manager = new NodeMetadataManager({
            rootDir,
            watch: effectiveWatch,
            formats: ['yaml', 'json', 'typescript', 'javascript']
        });

        // Initialize with default type registry
        this.manager.setTypeRegistry(DEFAULT_METADATA_TYPE_REGISTRY);
    }

    init = async (ctx: PluginContext) => {
        ctx.logger.info('Initializing Metadata Manager', {
            root: this.options.rootDir || process.cwd(),
            watch: this.options.watch,
            artifactSource: this.options.artifactSource?.mode,
        });

        // Register Metadata Manager as the primary metadata service provider.
        ctx.registerService('metadata', this.manager);
        console.log('[MetadataPlugin] Registered metadata service, has getRegisteredTypes:', typeof this.manager.getRegisteredTypes);

        // Register metadata system objects via the manifest service (if available).
        // MetadataPlugin may init before ObjectQLPlugin, so wrap in try/catch.
        // Skipped when `registerSystemObjects: false` (per-project kernels in
        // cloud / project mode — sys_* live exclusively in the control plane).
        const registerSysObjects = this.options.registerSystemObjects !== false;
        if (registerSysObjects) {
            try {
                const manifestService = ctx.getService<{ register(m: any): void }>('manifest');

                // Register the queryable metadata-layer platform objects.
                manifestService.register({
                    id: 'com.objectstack.metadata-objects',
                    name: 'Metadata Platform Objects',
                    version: '1.0.0',
                    type: 'plugin',
                    scope: 'system',
                    defaultDatasource: 'cloud',
                    objects: queryableMetadataObjects,
                });

                ctx.logger.info('Registered system metadata objects', {
                    queryable: queryableMetadataObjects.map((object) => object.name),
                });
            } catch {
                // ObjectQL not loaded yet — objects will be discovered via legacy fallback
            }
        }

        ctx.logger.info('MetadataPlugin providing metadata service (primary mode)', {
            mode: this.options.artifactSource?.mode ?? 'file-system',
            features: ['watch', 'multi-format', 'query', 'overlay', 'type-registry']
        });
    }

    start = async (ctx: PluginContext) => {
        const src = this.options.artifactSource;
        const mode = this.options.config?.bootstrap ?? 'eager';

        ctx.logger.info('[MetadataPlugin] Bootstrapping metadata', {
            bootstrap: mode,
            artifactSource: src?.mode ?? 'none',
        });

        if (mode === 'artifact-only') {
            // Sealed-runtime mode: ONLY load from a pre-compiled artifact. Never
            // touch the filesystem. Required for Edge / serverless / read-only
            // production deployments where the running process must not depend
            // on local source files.
            if (src?.mode === 'local-file') {
                await this._loadFromLocalFile(ctx, src.path, src.fetchTimeoutMs);
            } else if (src?.mode === 'artifact-api') {
                await this._loadFromArtifactApi(ctx, src);
            } else {
                throw new Error('[MetadataPlugin] bootstrap=artifact-only requires options.artifactSource to be set');
            }
        } else if (mode === 'lazy') {
            // On-demand mode: skip the eager filesystem priming pass entirely.
            // Reads go through MetadataManager.load*/list* which are backed by
            // the DatabaseLoader read-through cache and any registered loaders.
            // An artifact source, if present, is still honored so projects can
            // pin a known set of metadata at boot without paying the FS scan.
            if (src?.mode === 'local-file') {
                await this._loadFromLocalFile(ctx, src.path, src.fetchTimeoutMs);
            } else if (src?.mode === 'artifact-api') {
                await this._loadFromArtifactApi(ctx, src);
            } else {
                ctx.logger.info('[MetadataPlugin] lazy bootstrap — skipping filesystem priming; metadata loads on demand');
            }
        } else {
            // 'eager' (default): preserve historical behavior.
            if (src?.mode === 'local-file') {
                await this._loadFromLocalFile(ctx, src.path, src.fetchTimeoutMs);
            } else if (src?.mode === 'artifact-api') {
                await this._loadFromArtifactApi(ctx, src);
            } else {
                await this._loadFromFileSystem(ctx);
            }
        }

        // Bridge realtime service from kernel service registry to MetadataManager.
        try {
            const realtimeService = ctx.getService('realtime');
            if (realtimeService && typeof realtimeService === 'object' && 'publish' in realtimeService) {
                ctx.logger.info('[MetadataPlugin] Bridging realtime service to MetadataManager for event publishing');
                this.manager.setRealtimeService(realtimeService as any);
            }
        } catch (e: any) {
            ctx.logger.debug('[MetadataPlugin] No realtime service found — metadata events will not be published', {
                error: e.message,
            });
        }
    }

    /**
     * Fetch JSON content from a URL with configurable timeout.
     */
    private async _fetchJson(url: string, fetchTimeoutMs?: number, token?: string): Promise<unknown> {
        const envTimeout = Number(process.env.OS_ARTIFACT_FETCH_TIMEOUT_MS);
        const timeoutMs = fetchTimeoutMs
            ?? (Number.isFinite(envTimeout) && envTimeout > 0 ? envTimeout : undefined)
            ?? 60_000;
        const controller = new AbortController();
        const timer = timeoutMs > 0 ? setTimeout(() => controller.abort(), timeoutMs) : undefined;
        try {
            const headers: Record<string, string> = { Accept: 'application/json, */*;q=0.5' };
            if (token) headers.Authorization = `Bearer ${token}`;
            const res = await fetch(url, { redirect: 'follow', signal: controller.signal, headers });
            if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
            const content = await res.text();
            return JSON.parse(content);
        } catch (e: any) {
            if (e?.name === 'AbortError') {
                throw new Error(
                    `fetch timed out after ${timeoutMs}ms — set artifactSource.fetchTimeoutMs or OS_ARTIFACT_FETCH_TIMEOUT_MS to extend it (0 disables)`,
                );
            }
            throw e;
        } finally {
            if (timer) clearTimeout(timer);
        }
    }

    /**
     * Parse raw artifact JSON (envelope or bare definition) and register all
     * metadata items into the MetadataManager.
     */
    private async _parseAndRegisterArtifact(ctx: PluginContext, raw: unknown, label: string): Promise<number> {
        const { ProjectArtifactSchema } = await import('@objectstack/spec/cloud');
        const { ObjectStackDefinitionSchema } = await import('@objectstack/spec');

        let metadata: Record<string, unknown[]>;

        const obj = raw as any;
        if (obj?.schemaVersion && obj?.commitId && obj?.metadata !== undefined) {
            const artifact = ProjectArtifactSchema.parse(obj);
            metadata = artifact.metadata as Record<string, unknown[]>;
        } else if (obj?.success && obj?.data?.metadata) {
            // Unwrap cloud API envelope: { success: true, data: { metadata: {...} } }
            const artifact = ProjectArtifactSchema.parse(obj.data);
            metadata = artifact.metadata as Record<string, unknown[]>;
        } else {
            const def = ObjectStackDefinitionSchema.parse(obj);
            const canonical = JSON.stringify(def, Object.keys(def).sort());
            const checksum = createHash('sha256').update(canonical).digest('hex');
            const projectId = this.options.projectId ?? 'proj_local';
            ProjectArtifactSchema.parse({
                schemaVersion: '0.1',
                projectId,
                commitId: 'local-dev',
                checksum,
                metadata: def,
            });
            metadata = def as Record<string, unknown[]>;
        }

        const memLoader = new MemoryLoader();
        const manifestPackageId =
            (metadata as any)?.manifest?.id ?? (metadata as any)?.id ?? undefined;

        let totalRegistered = 0;
        for (const [field, metaType] of Object.entries(ARTIFACT_FIELD_TO_TYPE)) {
            const items = (metadata as any)[field];
            if (!Array.isArray(items) || items.length === 0) continue;
            for (const item of items) {
                const name = (item as any)?.name;
                if (!name) continue;
                if (manifestPackageId && (item as any)._packageId === undefined) {
                    (item as any)._packageId = manifestPackageId;
                }
                await memLoader.save(metaType, name, item);
                await this.manager.register(metaType, name, item);
                totalRegistered++;
            }
        }

        this.manager.registerLoader(memLoader);
        ctx.logger.info('[MetadataPlugin] Artifact metadata loaded', { source: label, totalRegistered });
        return totalRegistered;
    }

    private async _loadFromLocalFile(ctx: PluginContext, filePath: string, fetchTimeoutMs?: number): Promise<void> {
        const isUrl = /^https?:\/\//i.test(filePath);
        ctx.logger.info(
            `[MetadataPlugin] Loading metadata from ${isUrl ? 'remote URL' : 'local artifact file'}`,
            { path: filePath },
        );

        let raw: unknown;
        try {
            if (isUrl) {
                raw = await this._fetchJson(filePath, fetchTimeoutMs);
            } else {
                const content = await readFile(filePath, 'utf8');
                raw = JSON.parse(content);
            }
        } catch (e: any) {
            throw new Error(`[MetadataPlugin] Cannot read artifact ${isUrl ? 'URL' : 'file'} at "${filePath}": ${e.message}`);
        }

        await this._parseAndRegisterArtifact(ctx, raw, filePath);
    }

    /**
     * P2: Load metadata from the cloud artifact API endpoint.
     */
    private async _loadFromArtifactApi(
        ctx: PluginContext,
        src: { url: string; token?: string; commitId?: string; fetchTimeoutMs?: number },
    ): Promise<void> {
        const projectId = this.options.projectId;
        if (!projectId) {
            throw new Error('[MetadataPlugin] artifact-api source requires options.projectId to be set');
        }

        // Build the artifact URL:
        //   ${url}/api/v1/cloud/projects/${projectId}/artifact[?commit=${commitId}]
        let artifactUrl = src.url.replace(/\/+$/, '');
        // If the URL already contains /api/v1, use it as-is; otherwise append default path.
        if (!/\/api\/v\d+\/cloud\/projects\//i.test(artifactUrl)) {
            artifactUrl = `${artifactUrl}/api/v1/cloud/projects/${projectId}/artifact`;
        }
        if (src.commitId) {
            artifactUrl += `${artifactUrl.includes('?') ? '&' : '?'}commit=${encodeURIComponent(src.commitId)}`;
        }

        ctx.logger.info('[MetadataPlugin] Loading metadata from artifact API', { url: artifactUrl });

        let raw: unknown;
        try {
            raw = await this._fetchJson(artifactUrl, src.fetchTimeoutMs, src.token);
        } catch (e: any) {
            throw new Error(`[MetadataPlugin] Cannot load artifact from API "${artifactUrl}": ${e.message}`);
        }

        await this._parseAndRegisterArtifact(ctx, raw, artifactUrl);
    }

    private async _loadFromFileSystem(ctx: PluginContext): Promise<void> {
        ctx.logger.info('Loading metadata from file system...');

        const sortedTypes = [...DEFAULT_METADATA_TYPE_REGISTRY]
            .sort((a, b) => a.loadOrder - b.loadOrder);

        let totalLoaded = 0;
        for (const entry of sortedTypes) {
            try {
                const items = await this.manager.loadMany(entry.type, {
                    recursive: true,
                    patterns: entry.filePatterns,
                });

                if (items.length > 0) {
                    for (const item of items) {
                        const meta = item as any;
                        if (meta?.name) {
                            await this.manager.register(entry.type, meta.name, item);
                        }
                    }
                    ctx.logger.info(`Loaded ${items.length} ${entry.type} from file system`);
                    totalLoaded += items.length;
                }
            } catch (e: any) {
                ctx.logger.debug(`No ${entry.type} metadata found`, { error: e.message });
            }
        }

        ctx.logger.info('Metadata loading complete', {
            totalItems: totalLoaded,
            registeredTypes: sortedTypes.length,
        });
    }
}