diff --git a/.talismanrc b/.talismanrc index dbe7be68..848d28ea 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,12 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 9ad01ca900e007d8ea4f5ffcef9f166665aba852b3b9a0a5f4506c04fd362138 + - filename: packages/contentstack-asset-management/src/import/fields.ts + checksum: e46c3eb94bba78ae06af0139a1b0fd4113c3dcfe879ed40cfe77659781526d5c + - filename: packages/contentstack-asset-management/src/import/asset-types.ts + checksum: 3525703fd2ac0f7ab3e963966c0abfa53d87c09f73dabab0889c7f41a5f1b003 + - filename: packages/contentstack-import/src/types/default-config.ts + checksum: 1c09acba953cfd7058a3e0d63f0a9bfbb8f28e903538eaa015fdc611402bbd4f + - filename: packages/contentstack-asset-management/src/types/asset-management-api.ts + checksum: 9394e67f4d0dea9ad27a8592adf99441f40731b788335cd7699c78205bdaad58 + - filename: packages/contentstack-import/src/import/modules/assets.ts + checksum: 98cea49283eb5d975168dc51e27b0e2fb541a9ed9b4368206f99b434e104f096 version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 96ff8490..43c1b6d1 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -1,5 +1,36 @@ -export const BATCH_SIZE = 50; -export const CHUNK_FILE_SIZE_MB = 1; +/** Fallback when export/import do not pass `chunkWriteBatchSize`. */ +export const FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE = 50; +/** Fallback when export/import do not pass `chunkFileSizeMb`. */ +export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; +/** Fallback when import does not pass `apiConcurrency`. */ +export const FALLBACK_AM_API_CONCURRENCY = 5; +/** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ +export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; + +/** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ +export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'asset_types_count', +] as const; +export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'category', + 'preview_image_url', + 'category_detail', +] as const; + +/** @deprecated Use FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE */ +export const BATCH_SIZE = FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE; +/** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */ +export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB; /** * Main process name for Asset Management 2.0 export (single progress bar). @@ -17,10 +48,15 @@ export const PROCESS_NAMES = { AM_FIELDS: 'Fields', AM_ASSET_TYPES: 'Asset types', AM_DOWNLOADS: 'Asset downloads', + // Import process names + AM_IMPORT_FIELDS: 'Import fields', + AM_IMPORT_ASSET_TYPES: 'Import asset types', + AM_IMPORT_FOLDERS: 'Import folders', + AM_IMPORT_ASSETS: 'Import assets', } as const; /** - * Status messages for each process (exporting, fetching, failed). + * Status messages for each process (exporting, fetching, importing, failed). */ export const PROCESS_STATUS = { [PROCESS_NAMES.AM_SPACE_METADATA]: { @@ -47,4 +83,20 @@ export const PROCESS_STATUS = { DOWNLOADING: 'Downloading asset files...', FAILED: 'Failed to download assets.', }, + [PROCESS_NAMES.AM_IMPORT_FIELDS]: { + IMPORTING: 'Importing shared fields...', + FAILED: 'Failed to import fields.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSET_TYPES]: { + IMPORTING: 'Importing shared asset types...', + FAILED: 'Failed to import asset types.', + }, + [PROCESS_NAMES.AM_IMPORT_FOLDERS]: { + IMPORTING: 'Importing folders...', + FAILED: 'Failed to import folders.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSETS]: { + IMPORTING: 'Importing assets...', + FAILED: 'Failed to import assets.', + }, } as const; diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 140b5664..28016691 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -21,8 +21,8 @@ export default class ExportAssets extends AssetManagementExportAdapter { log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid), - this.getWorkspaceAssets(workspace.space_uid), + this.getWorkspaceFolders(workspace.space_uid, workspace.uid), + this.getWorkspaceAssets(workspace.space_uid, workspace.uid), ]); await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index 7521eae5..9990cf21 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,8 +5,11 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { AssetManagementAPIConfig } from '../types/asset-management-api'; import type { ExportContext } from '../types/export-types'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; -import { AM_MAIN_PROCESS_NAME } from '../constants/index'; -import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants/index'; +import { + AM_MAIN_PROCESS_NAME, + FALLBACK_AM_CHUNK_FILE_SIZE_MB, + FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE, +} from '../constants/index'; export type { ExportContext }; @@ -83,17 +86,19 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter { await writeFile(pResolve(dir, indexFileName), '{}'); return; } + const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; + const batchSize = this.exportContext.chunkWriteBatchSize ?? FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE; const fs = new FsUtility({ basePath: dir, indexFileName, - chunkFileSize: CHUNK_FILE_SIZE_MB, + chunkFileSize: chunkMb, moduleName, fileExt: 'json', metaPickKeys, keepMetadata: true, }); - for (let i = 0; i < items.length; i += BATCH_SIZE) { - const batch = items.slice(i, i + BATCH_SIZE); + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize); fs.writeIntoFile(batch as Record[], { mapKeyVal: true }); } fs.completeFile(true); diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index cf3ff2c3..cdc23c93 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -35,8 +35,11 @@ export class ExportSpaces { branchName, assetManagementUrl, org_uid, + apiKey, context, securedAssets, + chunkWriteBatchSize, + chunkFileSizeMb, } = this.options; if (!linkedWorkspaces.length) { @@ -54,7 +57,9 @@ export class ExportSpaces { const totalSteps = 2 + linkedWorkspaces.length * 4; const progress = this.createProgress(); progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); - progress.startProcess(AM_MAIN_PROCESS_NAME).updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); + progress + .startProcess(AM_MAIN_PROCESS_NAME) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); const apiConfig: AssetManagementAPIConfig = { baseURL: assetManagementUrl, @@ -65,6 +70,8 @@ export class ExportSpaces { spacesRootPath, context, securedAssets, + chunkWriteBatchSize, + chunkFileSizeMb, }; const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts new file mode 100644 index 00000000..cb23b617 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -0,0 +1,101 @@ +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; + +/** + * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs + * each to the target org-level AM endpoint (`POST /api/asset_types`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch asset types that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new asset types. + */ +export default class ImportAssetTypes extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS]; + const dir = this.getAssetTypesDir(); + const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json'; + const items = await readChunkedJsonItems>(dir, indexName, this.importContext.context); + + if (items.length === 0) { + log.debug('No shared asset types to import', this.importContext.context); + return; + } + + // Fetch existing asset types from the target org keyed by uid for diff comparison. + // Asset types are org-level; the spaceUid param in getWorkspaceAssetTypes is unused in the path. + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceAssetTypes(''); + for (const at of existing.asset_types ?? []) { + existingByUid.set(at.uid, at as Record); + } + log.debug(`Target org has ${existingByUid.size} existing asset type(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context); + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + + type ToCreate = { uid: string; payload: Record }; + const toCreate: ToCreate[] = []; + + for (const assetType of items) { + const uid = assetType.uid as string; + + if (assetType.is_system) { + log.debug(`Skipping system asset type: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(assetType, stripKeys); + const existingClean = omit(existing, stripKeys); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Asset type "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `asset-type: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + continue; + } + + toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record }); + } + + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { + try { + await this.createAssetType(payload as any); + this.tick(true, `asset-type: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Imported asset type: ${uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset-type: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, + ); + log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts new file mode 100644 index 00000000..4b942a18 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -0,0 +1,248 @@ +import { resolve as pResolve, join } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; +import { runInBatches } from '../utils/concurrent-batch'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +type FolderRecord = { + uid: string; + title: string; + description?: string; + parent_uid?: string; +}; + +type AssetRecord = { + uid: string; + url: string; + filename?: string; + file_name?: string; + parent_uid?: string; + title?: string; + description?: string; +}; + +/** + * Imports folders and assets for a single AM space. + * - Reads `spaces/{spaceUid}/assets/folders.json` → creates folders, builds folderUidMap + * - Reads chunked `assets.json` → uploads each file from `files/{oldUid}/{filename}` + * - Builds UID and URL mapper entries for entries.ts consumption + * Mirrors ExportAssets. + */ +export default class ImportAssets extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + /** + * Loads chunked `assets.json` when present; shared by reuse (identity maps) and upload path. + */ + private async loadExportedAssetItems(spaceDir: string): Promise { + const assetsDir = pResolve(spaceDir, 'assets'); + const assetsIndex = this.importContext.assetsFileName ?? 'assets.json'; + if (!existsSync(join(assetsDir, assetsIndex))) { + return null; + } + return readChunkedJsonItems(assetsDir, assetsIndex, this.importContext.context); + } + + /** + * Build identity uid/url mappers from export JSON only (reuse path — no upload). + * Keys and values are equal so lookupAssets contract is satisfied without remapping. + */ + async buildIdentityMappersFromExport( + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const uidMap: Record = {}; + const urlMap: Record = {}; + + const assetItems = await this.loadExportedAssetItems(spaceDir); + if (!assetItems) { + log.debug( + `No assets.json index in ${pResolve(spaceDir, 'assets')}, identity mappers empty`, + this.importContext.context, + ); + return { uidMap, urlMap }; + } + log.debug( + `Building identity mappers for ${assetItems.length} exported asset(s) (reuse path)`, + this.importContext.context, + ); + + for (const asset of assetItems) { + if (asset.uid) { + uidMap[asset.uid] = asset.uid; + } + if (asset.url) { + urlMap[asset.url] = asset.url; + } + } + + return { uidMap, urlMap }; + } + + async start( + newSpaceUid: string, + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const assetsDir = pResolve(spaceDir, 'assets'); + const uidMap: Record = {}; + const urlMap: Record = {}; + + // ----------------------------------------------------------------------- + // 1. Import folders + // ----------------------------------------------------------------------- + const folderUidMap: Record = {}; + const foldersFileName = this.importContext.foldersFileName ?? 'folders.json'; + const foldersFilePath = join(assetsDir, foldersFileName); + + if (existsSync(foldersFilePath)) { + let foldersData: unknown; + try { + foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8')); + } catch (e) { + log.debug(`Could not read ${foldersFileName}: ${e}`, this.importContext.context); + } + + if (foldersData) { + const folders = getArrayFromResponse(foldersData, 'folders') as FolderRecord[]; + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Importing ${folders.length} folder(s) for space ${newSpaceUid}`, this.importContext.context); + await this.importFolders(newSpaceUid, folders, folderUidMap); + } + } + + // ----------------------------------------------------------------------- + // 2. Import assets (chunked) + // ----------------------------------------------------------------------- + const assetItems = await this.loadExportedAssetItems(spaceDir); + if (!assetItems) { + log.debug(`No assets.json index found in ${assetsDir}, skipping asset upload`, this.importContext.context); + return { uidMap, urlMap }; + } + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploading ${assetItems.length} asset(s) for space ${newSpaceUid}`, this.importContext.context); + + type UploadJob = { + asset: AssetRecord; + filePath: string; + mappedParentUid: string | undefined; + oldUid: string; + }; + const uploadJobs: UploadJob[] = []; + + for (const asset of assetItems) { + const oldUid = asset.uid; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + const filePath = pResolve(assetsDir, 'files', oldUid, filename); + + if (!existsSync(filePath)) { + log.debug(`Asset file not found: ${filePath}, skipping`, this.importContext.context); + this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); + continue; + } + + const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; + const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + + uploadJobs.push({ asset, filePath, mappedParentUid, oldUid }); + } + + await runInBatches(uploadJobs, this.apiConcurrency, async ({ asset, filePath, mappedParentUid, oldUid }) => { + const filename = asset.filename ?? asset.file_name ?? 'asset'; + try { + const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { + title: asset.title ?? filename, + description: asset.description, + parent_uid: mappedParentUid, + }); + + uidMap[oldUid] = created.uid; + + if (asset.url && created.url) { + urlMap[asset.url] = created.url; + } + + this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploaded asset ${oldUid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset: ${oldUid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSETS, + ); + log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + } + }); + + return { uidMap, urlMap }; + } + + /** + * Creates folders respecting hierarchy: parents before children. + * Uses multiple passes to handle arbitrary depth without requiring sorted input. + */ + private async importFolders( + newSpaceUid: string, + folders: FolderRecord[], + folderUidMap: Record, + ): Promise { + let remaining = [...folders]; + let prevLength = -1; + + while (remaining.length > 0 && remaining.length !== prevLength) { + prevLength = remaining.length; + const ready: FolderRecord[] = []; + const nextPass: FolderRecord[] = []; + + for (const folder of remaining) { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; + const parentMapped = isRootParent || folderUidMap[parentUid] !== undefined; + + if (!parentMapped) { + nextPass.push(folder); + } else { + ready.push(folder); + } + } + + await runInBatches(ready, this.apiConcurrency, async (folder) => { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; + try { + const { folder: created } = await this.createFolder(newSpaceUid, { + title: folder.title, + description: folder.description, + parent_uid: isRootParent ? undefined : folderUidMap[parentUid!], + }); + folderUidMap[folder.uid] = created.uid; + this.tick(true, `folder: ${folder.uid}`, null, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Created folder ${folder.uid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `folder: ${folder.uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].FAILED, + PROCESS_NAMES.AM_IMPORT_FOLDERS, + ); + log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); + } + }); + + remaining = nextPass; + } + + if (remaining.length > 0) { + log.debug( + `${remaining.length} folder(s) could not be imported (unresolved parent UIDs)`, + this.importContext.context, + ); + } + } +} diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts new file mode 100644 index 00000000..298f7f61 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -0,0 +1,84 @@ +import { resolve as pResolve } from 'node:path'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY } from '../constants/index'; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; + +export type { ImportContext }; + +/** + * Base class for all AM 2.0 import modules. Mirrors AssetManagementExportAdapter + * but carries ImportContext (spacesRootPath, apiKey, host, etc.) instead of ExportContext. + */ +export class AssetManagementImportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly importContext: ImportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.importContext = importContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.importContext.spacesRootPath; + } + + /** Parallel AM API limit for import batches. */ + protected get apiConcurrency(): number { + return this.importContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY; + } + + protected getAssetTypesDir(): string { + return pResolve(this.importContext.spacesRootPath, this.importContext.assetTypesDir ?? 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.importContext.spacesRootPath, this.importContext.fieldsDir ?? 'fields'); + } + + /** + * Reads all items from a chunked JSON store via {@link readChunkedJsonItems} (FsUtility). + */ + protected async readAllChunkedJson>(dir: string, indexFileName: string): Promise { + return readChunkedJsonItems(dir, indexFileName, this.importContext.context); + } +} diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts new file mode 100644 index 00000000..a153eb31 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -0,0 +1,101 @@ +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { FALLBACK_FIELDS_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; +import { readChunkedJsonItems } from '../utils/chunked-json-read'; + +/** + * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the + * target org-level AM fields endpoint (`POST /api/fields`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch fields that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new fields. + */ +export default class ImportFields extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const stripKeys = this.importContext.fieldsImportInvalidKeys ?? [...FALLBACK_FIELDS_IMPORT_INVALID_KEYS]; + const dir = this.getFieldsDir(); + const indexName = this.importContext.fieldsFileName ?? 'fields.json'; + const items = await readChunkedJsonItems>(dir, indexName, this.importContext.context); + + if (items.length === 0) { + log.debug('No shared fields to import', this.importContext.context); + return; + } + + // Fetch existing fields from the target org keyed by uid for diff comparison. + // Fields are org-level; the spaceUid param in getWorkspaceFields is unused in the path. + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceFields(''); + for (const f of existing.fields ?? []) { + existingByUid.set(f.uid, f as Record); + } + log.debug(`Target org has ${existingByUid.size} existing field(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing fields, will attempt to create all: ${e}`, this.importContext.context); + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + + type ToCreate = { uid: string; payload: Record }; + const toCreate: ToCreate[] = []; + + for (const field of items) { + const uid = field.uid as string; + + if (field.is_system) { + log.debug(`Skipping system field: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(field, stripKeys); + const existingClean = omit(existing, stripKeys); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Field "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `field: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + continue; + } + + toCreate.push({ uid, payload: omit(field, stripKeys) as Record }); + } + + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { + try { + await this.createField(payload as any); + this.tick(true, `field: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Imported field: ${uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `field: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, + PROCESS_NAMES.AM_IMPORT_FIELDS, + ); + log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/import/index.ts b/packages/contentstack-asset-management/src/import/index.ts new file mode 100644 index 00000000..61d8a457 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/index.ts @@ -0,0 +1,7 @@ +export { ImportSpaces } from './spaces'; +export { default as ImportWorkspace } from './workspaces'; +export { default as ImportAssets } from './assets'; +export { default as ImportFields } from './fields'; +export { default as ImportAssetTypes } from './asset-types'; +export { AssetManagementImportAdapter } from './base'; +export type { ImportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts new file mode 100644 index 00000000..d44fc795 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -0,0 +1,218 @@ +import { resolve as pResolve, join } from 'node:path'; +import { mkdirSync, readdirSync, statSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + AssetManagementImportOptions, + ImportContext, + ImportResult, + SpaceMapping, +} from '../types/asset-management-api'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import ImportAssetTypes from './asset-types'; +import ImportFields from './fields'; +import ImportWorkspace from './workspaces'; + +/** + * Top-level orchestrator for AM 2.0 import. + * Mirrors ExportSpaces: creates shared fields + asset types, then imports each space. + * Returns combined uidMap, urlMap, and spaceMappings for the bridge module. + */ +export class ImportSpaces { + private readonly options: AssetManagementImportOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: AssetManagementImportOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const { + contentDir, + assetManagementUrl, + org_uid, + apiKey, + host, + sourceApiKey, + context, + apiConcurrency, + spacesDirName, + fieldsDir, + assetTypesDir, + fieldsFileName, + assetTypesFileName, + foldersFileName, + assetsFileName, + fieldsImportInvalidKeys, + assetTypesImportInvalidKeys, + mapperRootDir, + mapperAssetsModuleDir, + mapperUidFileName, + mapperUrlFileName, + mapperSpaceUidFileName, + } = this.options; + + const spacesRootPath = pResolve(contentDir, spacesDirName ?? 'spaces'); + + const importContext: ImportContext = { + spacesRootPath, + sourceApiKey, + apiKey, + host, + org_uid, + context, + apiConcurrency, + spacesDirName, + fieldsDir, + assetTypesDir, + fieldsFileName, + assetTypesFileName, + foldersFileName, + assetsFileName, + fieldsImportInvalidKeys, + assetTypesImportInvalidKeys, + mapperRootDir, + mapperAssetsModuleDir, + mapperUidFileName, + mapperUrlFileName, + mapperSpaceUidFileName, + }; + + const apiConfig: AssetManagementAPIConfig = { + baseURL: assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + + // Discover space directories + let spaceDirs: string[] = []; + try { + spaceDirs = readdirSync(spacesRootPath).filter((entry) => { + try { + return statSync(join(spacesRootPath, entry)).isDirectory() && entry.startsWith('am'); + } catch { + return false; + } + }); + } catch (e) { + log.debug(`Could not read spaces root path ${spacesRootPath}: ${e}`, context); + } + + const totalSteps = 2 + spaceDirs.length * 2; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress.startProcess(AM_MAIN_PROCESS_NAME); + + const allUidMap: Record = {}; + const allUrlMap: Record = {}; + const allSpaceUidMap: Record = {}; + const spaceMappings: SpaceMapping[] = []; + let hasFailures = false; + + // Space UIDs already present in the target org — reuse when export dir name matches a uid here. + const existingSpaceUids = new Set(); + try { + const adapterForList = new AssetManagementAdapter(apiConfig); + await adapterForList.init(); + const { spaces } = await adapterForList.listSpaces(); + for (const s of spaces) { + if (s.uid) existingSpaceUids.add(s.uid); + } + log.debug(`Found ${existingSpaceUids.size} existing space uid(s) in target org`, context); + } catch (e) { + log.debug(`Could not fetch existing spaces — reuse-by-uid disabled: ${e}`, context); + } + + try { + // 1. Import shared fields + progress.updateStatus(`Importing shared fields...`, AM_MAIN_PROCESS_NAME); + const fieldsImporter = new ImportFields(apiConfig, importContext); + fieldsImporter.setParentProgressManager(progress); + await fieldsImporter.start(); + + // 2. Import shared asset types + progress.updateStatus('Importing shared asset types...', AM_MAIN_PROCESS_NAME); + const assetTypesImporter = new ImportAssetTypes(apiConfig, importContext); + assetTypesImporter.setParentProgressManager(progress); + await assetTypesImporter.start(); + + // 3. Import each space — continue on failure so partially-imported data is never lost + for (const spaceUid of spaceDirs) { + const spaceDir = join(spacesRootPath, spaceUid); + progress.updateStatus(`Importing space: ${spaceUid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Importing space: ${spaceUid}`, context); + + try { + const workspaceImporter = new ImportWorkspace(apiConfig, importContext); + workspaceImporter.setParentProgressManager(progress); + const result = await workspaceImporter.start(spaceUid, spaceDir, existingSpaceUids); + + // Newly created spaces get a new uid — add so later iterations in this run see it. + existingSpaceUids.add(result.newSpaceUid); + + Object.assign(allUidMap, result.uidMap); + Object.assign(allUrlMap, result.urlMap); + allSpaceUidMap[result.oldSpaceUid] = result.newSpaceUid; + spaceMappings.push({ + oldSpaceUid: result.oldSpaceUid, + newSpaceUid: result.newSpaceUid, + workspaceUid: result.workspaceUid, + isDefault: result.isDefault, + }); + + log.debug(`Imported space ${spaceUid} → ${result.newSpaceUid}`, context); + } catch (err) { + hasFailures = true; + progress.tick( + false, + `space: ${spaceUid}`, + (err as Error)?.message ?? 'Failed to import space', + AM_MAIN_PROCESS_NAME, + ); + log.debug(`Failed to import space ${spaceUid}: ${err}`, context); + } + } + + if (this.options.backupDir) { + const mapperRoot = this.options.mapperRootDir ?? 'mapper'; + const mapperAssetsMod = this.options.mapperAssetsModuleDir ?? 'assets'; + const mapperDir = join(this.options.backupDir, mapperRoot, mapperAssetsMod); + mkdirSync(mapperDir, { recursive: true }); + const uidFile = this.options.mapperUidFileName ?? 'uid-mapping.json'; + const urlFile = this.options.mapperUrlFileName ?? 'url-mapping.json'; + const spaceUidFile = this.options.mapperSpaceUidFileName ?? 'space-uid-mapping.json'; + await writeFile(join(mapperDir, uidFile), JSON.stringify(allUidMap), 'utf8'); + await writeFile(join(mapperDir, urlFile), JSON.stringify(allUrlMap), 'utf8'); + await writeFile(join(mapperDir, spaceUidFile), JSON.stringify(allSpaceUidMap), 'utf8'); + log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', context); + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, !hasFailures); + log.debug('Asset Management 2.0 import completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + throw err; + } + + return { uidMap: allUidMap, urlMap: allUrlMap, spaceMappings, spaceUidMap: allSpaceUidMap }; + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} diff --git a/packages/contentstack-asset-management/src/import/workspaces.ts b/packages/contentstack-asset-management/src/import/workspaces.ts new file mode 100644 index 00000000..5d13450d --- /dev/null +++ b/packages/contentstack-asset-management/src/import/workspaces.ts @@ -0,0 +1,82 @@ +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext, SpaceMapping } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import ImportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +type WorkspaceResult = SpaceMapping & { + uidMap: Record; + urlMap: Record; +}; + +/** + * Handles import for a single AM 2.0 space directory. + * Reads `metadata.json`, creates the space in the target org when its uid is not + * already present, or reuses the existing space and emits identity mappers only. + * Returns the SpaceMapping plus UID/URL maps for the mapper files. + */ +export default class ImportWorkspace extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start( + oldSpaceUid: string, + spaceDir: string, + existingSpaceUids: Set = new Set(), + ): Promise { + await this.init(); + + // Read exported metadata + const metadataPath = join(spaceDir, 'metadata.json'); + let metadata: Record = {}; + try { + metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as Record; + } catch (e) { + log.debug(`Could not read metadata.json for space ${oldSpaceUid}: ${e}`, this.importContext.context); + } + + const exportedTitle = (metadata.title as string) ?? oldSpaceUid; + const description = metadata.description as string | undefined; + const isDefault = (metadata.is_default as boolean) ?? false; + const workspaceUid = 'main'; + + const assetsImporter = new ImportAssets(this.apiConfig, this.importContext); + if (this.progressOrParent) assetsImporter.setParentProgressManager(this.progressOrParent); + + // Reuse: target org already has a space with the same uid as the export directory. + if (existingSpaceUids.has(oldSpaceUid)) { + log.info( + `Reusing existing AM space "${oldSpaceUid}" (uid matches export directory); skipping create and upload.`, + this.importContext.context, + ); + const newSpaceUid = oldSpaceUid; + const { uidMap, urlMap } = await assetsImporter.buildIdentityMappersFromExport(spaceDir); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid} (reused)`, null, PROCESS_NAMES.AM_SPACE_METADATA); + return { + oldSpaceUid, + newSpaceUid, + workspaceUid, + isDefault, + uidMap, + urlMap, + }; + } + + // Create new space with exact exported title + log.debug(`Creating space "${exportedTitle}" (old uid: ${oldSpaceUid})`, this.importContext.context); + + const { space } = await this.createSpace({ title: exportedTitle, description }); + const newSpaceUid = space.uid; + + log.debug(`Created space ${newSpaceUid} (old: ${oldSpaceUid})`, this.importContext.context); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid}`, null, PROCESS_NAMES.AM_SPACE_METADATA); + + const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); + + return { oldSpaceUid, newSpaceUid, workspaceUid, isDefault, uidMap, urlMap }; + } +} diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts index f0ff59bd..c66c638d 100644 --- a/packages/contentstack-asset-management/src/index.ts +++ b/packages/contentstack-asset-management/src/index.ts @@ -2,3 +2,4 @@ export * from './constants/index'; export * from './types'; export * from './utils'; export * from './export'; +export * from './import'; diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 733ecada..8ae90f6a 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -36,6 +36,9 @@ export type Space = { /** Response shape of GET /api/spaces/{space_uid}. */ export type SpaceResponse = { space: Space }; +/** Response shape of GET /api/spaces (list all spaces in the org). */ +export type SpacesListResponse = { spaces: Space[]; count?: number }; + /** * Field structure from GET /api/fields (org-level). */ @@ -118,10 +121,11 @@ export type AssetManagementAPIConfig = { */ export interface IAssetManagementAdapter { init(): Promise; + listSpaces(): Promise; getSpace(spaceUid: string): Promise; getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string): Promise; - getWorkspaceFolders(spaceUid: string): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; } @@ -137,4 +141,168 @@ export type AssetManagementExportOptions = { context?: Record; /** When true, the AM package will add authtoken to asset download URLs. */ securedAssets?: boolean; + /** + * API key of the stack being exported. + * Saved to `spaces/export-metadata.json` so that during import the URL mapper + * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). + */ + apiKey?: string; + /** + * Chunked JSON write batch size (items per FsUtility write). From export `modules['asset-management']`. + */ + chunkWriteBatchSize?: number; + /** + * FsUtility `chunkFileSize` in MB for AM export chunked writes. + */ + chunkFileSizeMb?: number; +}; + +// --------------------------------------------------------------------------- +// Import types +// --------------------------------------------------------------------------- + +/** + * Context passed down to every import adapter class. + * Mirrors ExportContext but carries the import-specific fields needed for + * URL mapper reconstruction and API calls. + */ +export type ImportContext = { + /** Absolute path to the root `spaces/` directory inside the backup/content dir. */ + spacesRootPath: string; + /** Source stack API key — used to reconstruct old CMA proxy URLs. */ + sourceApiKey?: string; + /** Target stack API key — used to build new CMA proxy URLs. */ + apiKey: string; + /** Target CMA host (may include /v3), e.g. "https://api.contentstack.io/v3". */ + host: string; + /** Target org UID — required as `x-organization-uid` header when creating spaces. */ + org_uid: string; + /** Optional logging context (same shape as ExportConfig.context). */ + context?: Record; + /** + * Max parallel AM API calls for import (fields, asset types, folders batch, uploads). + * Set from `AssetManagementImportOptions.apiConcurrency`. + */ + apiConcurrency?: number; + /** Relative dir under content dir for AM export root (e.g. `spaces`). */ + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + /** `{backupDir}/{mapperRootDir}/{mapperAssetsModuleDir}/` for AM mapper JSON. */ + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; +}; + +/** + * Options accepted by the top-level `ImportSpaces` class. + */ +export type AssetManagementImportOptions = { + /** Absolute path to the root content / backup directory. */ + contentDir: string; + /** AM 2.0 base URL (e.g. "https://am.contentstack.io"). */ + assetManagementUrl: string; + /** Target organisation UID. */ + org_uid: string; + /** Target stack API key. */ + apiKey: string; + /** Target CMA host. */ + host: string; + /** Source stack API key — used for old CMA proxy URL reconstruction. */ + sourceApiKey?: string; + /** Optional logging context. */ + context?: Record; + /** + * When set, mapper files are written under `{backupDir}/mapper/assets/` after import. + */ + backupDir?: string; + /** Parallel AM API limit; defaults to package constant when omitted. */ + apiConcurrency?: number; + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; +}; + +/** + * Maps an old source-org space UID to the newly created target-org space UID. + */ +export type SpaceMapping = { + oldSpaceUid: string; + newSpaceUid: string; + /** Workspace identifier inside the space (typically "main"). */ + workspaceUid: string; + isDefault: boolean; +}; + +/** + * The value returned by `ImportSpaces.start()`. + * When `backupDir` is set on options, the AM package also writes these maps under + * `mapper/assets/` for `entries.ts` to resolve asset references. + */ +export type ImportResult = { + uidMap: Record; + urlMap: Record; + spaceMappings: SpaceMapping[]; + /** old space UID → new space UID, written to mapper/assets/space-uid-mapping.json */ + spaceUidMap: Record; +}; + +// --------------------------------------------------------------------------- +// Import payload types (confirmed from Postman collection) +// --------------------------------------------------------------------------- + +export type CreateSpacePayload = { + title: string; + description?: string; +}; + +export type CreateFolderPayload = { + title: string; + description?: string; + parent_uid?: string; +}; + +export type CreateAssetMetadata = { + title?: string; + description?: string; + parent_uid?: string; +}; + +export type CreateFieldPayload = { + uid: string; + title: string; + display_type?: string; + child?: unknown[]; + is_mandatory?: boolean; + is_multiple?: boolean; + [key: string]: unknown; +}; + +export type CreateAssetTypePayload = { + uid: string; + title: string; + description?: string; + content_type?: string; + file_extension?: string | string[]; + fields?: string[]; + [key: string]: unknown; }; diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index 25b8dcac..d1480adb 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -2,6 +2,9 @@ export type ExportContext = { spacesRootPath: string; context?: Record; securedAssets?: boolean; + /** From export config; AM falls back to package constants when unset. */ + chunkWriteBatchSize?: number; + chunkFileSizeMb?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts index b159cc33..b26b2664 100644 --- a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts @@ -1,11 +1,20 @@ +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; import { HttpClient, log, authenticationHandler } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, AssetTypesResponse, + CreateAssetMetadata, + CreateAssetTypePayload, + CreateFieldPayload, + CreateFolderPayload, + CreateSpacePayload, FieldsResponse, IAssetManagementAdapter, + Space, SpaceResponse, + SpacesListResponse, } from '../types/asset-management-api'; export class AssetManagementAdapter implements IAssetManagementAdapter { @@ -89,6 +98,13 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { } } + async listSpaces(): Promise { + log.debug('Fetching all spaces in org', this.config.context); + const result = await this.getSpaceLevel('', '/api/spaces', {}); + log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); + return result; + } + async getSpace(spaceUid: string): Promise { log.debug(`Fetching space: ${spaceUid}`, this.config.context); const path = `/api/spaces/${spaceUid}`; @@ -114,27 +130,30 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { spaceUid: string, path: string, logLabel: string, + queryParams: Record = {}, ): Promise { log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, path, {}); + const result = await this.getSpaceLevel(spaceUid, path, queryParams); const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); return result; } - async getWorkspaceAssets(spaceUid: string): Promise { + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', + workspaceUid ? { workspace: workspaceUid } : {}, ); } - async getWorkspaceFolders(spaceUid: string): Promise { + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', + workspaceUid ? { workspace: workspaceUid } : {}, ); } @@ -146,4 +165,118 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); return result; } + + // --------------------------------------------------------------------------- + // POST helpers + // --------------------------------------------------------------------------- + + /** + * Build headers for outgoing POST requests. + */ + private async getPostHeaders(extraHeaders: Record = {}): Promise> { + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + const authHeader: Record = authenticationHandler.isOauthEnabled + ? { authorization: token } + : { access_token: token }; + return { + Accept: 'application/json', + 'x-cs-api-version': '4', + ...(this.config.headers ?? {}), + ...authHeader, + ...extraHeaders, + }; + } + + private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); + log.debug(`POST ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + private async postMultipart(path: string, form: FormData, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders(extraHeaders); + log.debug(`POST (multipart) ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: form, + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API multipart POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + // --------------------------------------------------------------------------- + // Import API methods + // --------------------------------------------------------------------------- + + /** + * POST /api/spaces — creates a new space in the target org. + */ + async createSpace(payload: CreateSpacePayload): Promise<{ space: Space }> { + const orgUid = (this.config.headers as Record | undefined)?.organization_uid ?? ''; + return this.postJson<{ space: Space }>('/api/spaces', payload, { + 'x-organization-uid': orgUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/folders — creates a folder inside a space. + */ + async createFolder(spaceUid: string, payload: CreateFolderPayload): Promise<{ folder: { uid: string } }> { + return this.postJson<{ folder: { uid: string } }>(`/api/spaces/${encodeURIComponent(spaceUid)}/folders`, payload, { + space_key: spaceUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/assets — uploads an asset file as multipart form-data. + */ + async uploadAsset( + spaceUid: string, + filePath: string, + metadata: CreateAssetMetadata, + ): Promise<{ asset: { uid: string; url: string } }> { + const filename = basename(filePath); + const fileBuffer = readFileSync(filePath); + const blob = new Blob([fileBuffer]); + const form = new FormData(); + form.append('file', blob, filename); + if (metadata.title) form.append('title', metadata.title); + if (metadata.description) form.append('description', metadata.description); + if (metadata.parent_uid) form.append('parent_uid', metadata.parent_uid); + return this.postMultipart<{ asset: { uid: string; url: string } }>( + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + form, + { space_key: spaceUid }, + ); + } + + /** + * POST /api/fields — creates a shared field. + */ + async createField(payload: CreateFieldPayload): Promise<{ field: { uid: string } }> { + return this.postJson<{ field: { uid: string } }>('/api/fields', payload); + } + + /** + * POST /api/asset_types — creates a shared asset type. + */ + async createAssetType(payload: CreateAssetTypePayload): Promise<{ asset_type: { uid: string } }> { + return this.postJson<{ asset_type: { uid: string } }>('/api/asset_types', payload); + } } diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-read.ts b/packages/contentstack-asset-management/src/utils/chunked-json-read.ts new file mode 100644 index 00000000..4e09616c --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/chunked-json-read.ts @@ -0,0 +1,30 @@ +import { FsUtility, log } from '@contentstack/cli-utilities'; + +/** + * Read all items from a chunked JSON store (index + chunk files) using FsUtility, + * matching the pattern used in contentstack-import entry modules. + */ +export async function readChunkedJsonItems>( + basePath: string, + indexFileName: string, + context?: Record, +): Promise { + try { + const fs = new FsUtility({ basePath, indexFileName }); + const indexer = fs.indexFileContent; + const items: T[] = []; + for (const _ in indexer) { + const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => { + log.debug(`Error reading chunk: ${err}`, context); + return null; + }); + if (chunk) { + items.push(...(Object.values(chunk as Record))); + } + } + return items; + } catch (err) { + log.debug(`readChunkedJsonItems failed for ${basePath}/${indexFileName}: ${err}`, context); + return []; + } +} diff --git a/packages/contentstack-asset-management/src/utils/concurrent-batch.ts b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts new file mode 100644 index 00000000..dcd916c4 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts @@ -0,0 +1,34 @@ +/** + * Split an array into chunks of at most `size` elements. + */ +export function chunkArray(items: T[], size: number): T[][] { + if (size <= 0) { + return items.length ? [items] : []; + } + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) { + out.push(items.slice(i, i + size)); + } + return out; +} + +/** + * Run async work in batches of at most `concurrency` parallel tasks at a time. + * Uses Promise.allSettled per batch so one failure does not abort the batch. + */ +export async function runInBatches( + items: T[], + concurrency: number, + fn: (item: T, index: number) => Promise, +): Promise { + if (items.length === 0) { + return; + } + const limit = Math.max(1, concurrency); + const batches = chunkArray(items, limit); + let offset = 0; + for (const batch of batches) { + await Promise.allSettled(batch.map((item, j) => fn(item, offset + j))); + offset += batch.length; + } +} diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts index 0837e33f..b6925a87 100644 --- a/packages/contentstack-asset-management/src/utils/index.ts +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -1,8 +1,15 @@ export { AssetManagementAdapter } from './asset-management-api-adapter'; -export { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants'; +export { + BATCH_SIZE, + CHUNK_FILE_SIZE_MB, + FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE, + FALLBACK_AM_CHUNK_FILE_SIZE_MB, +} from '../constants'; +export { readChunkedJsonItems } from './chunked-json-read'; export { getArrayFromResponse, getAssetItems, getReadableStreamFromDownloadResponse, writeStreamToFile, } from './export-helpers'; +export { chunkArray, runInBatches } from './concurrent-batch'; diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index 5f3e14c4..ef745940 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -8,6 +8,7 @@ "@contentstack/cli-command": "~2.0.0-beta.4", "@contentstack/cli-utilities": "~2.0.0-beta.4", "@contentstack/cli-variants": "~2.0.0-beta.10", + "@contentstack/cli-asset-management": "~1.0.0", "@oclif/core": "^4.8.0", "async": "^3.2.6", "big-json": "^3.2.0", diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 14fe590b..788e7604 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -112,6 +112,10 @@ const config: DefaultConfig = { enableDownloadStatus: false, includeVersionedAssets: false, }, + 'asset-management': { + chunkWriteBatchSize: 50, + chunkFileSizeMb: 1, + }, content_types: { dirName: 'content_types', fileName: 'content_types.json', diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 01f785bf..bf8ff193 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -69,14 +69,18 @@ export default class ExportAssets extends BaseClass { this.exportConfig.org_uid = this.exportConfig.org_uid || (await getOrgUid(this.exportConfig)); const progress = this.createNestedProgress(this.currentModuleName); try { + const assetManagementModuleConfig = this.exportConfig.modules['asset-management']; const exporter = new ExportSpaces({ linkedWorkspaces, exportDir: this.exportConfig.exportDir, branchName: this.exportConfig.branchName || 'main', assetManagementUrl, org_uid: this.exportConfig.org_uid ?? '', + apiKey: this.exportConfig.apiKey, context: this.exportConfig.context as unknown as Record, securedAssets: this.exportConfig.securedAssets, + chunkWriteBatchSize: assetManagementModuleConfig?.chunkWriteBatchSize, + chunkFileSizeMb: assetManagementModuleConfig?.chunkFileSizeMb, }); exporter.setParentProgressManager(progress); await exporter.start(); diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index b0823992..30c58e06 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -96,6 +96,13 @@ export default interface DefaultConfig { includeVersionedAssets: boolean; dependencies?: Modules[]; }; + 'asset-management': { + /** Batch size for AM 2.0 chunked JSON writes (shared fields / asset types / etc.). */ + chunkWriteBatchSize: number; + /** Passed to FsUtility chunkFileSize (MB) when writing chunked export JSON. */ + chunkFileSizeMb: number; + dependencies?: Modules[]; + }; content_types: { dirName: string; fileName: string; diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index aad5ec76..f0af207d 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -9,6 +9,7 @@ "@contentstack/cli-command": "~2.0.0-beta.4", "@contentstack/cli-utilities": "~2.0.0-beta.4", "@contentstack/cli-variants": "~2.0.0-beta.9", + "@contentstack/cli-asset-management": "1.0.0", "@oclif/core": "^4.3.0", "big-json": "^3.2.0", "bluebird": "^3.7.2", diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index a18d7d07..a15332ba 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -101,6 +101,40 @@ const config: DefaultConfig = { folderValidKeys: ['name', 'parent_uid'], validKeys: ['title', 'parent_uid', 'description', 'tags'], }, + 'asset-management': { + dirName: 'spaces', + fieldsDir: 'fields', + assetTypesDir: 'asset_types', + fieldsFileName: 'fields.json', + assetTypesFileName: 'asset-types.json', + foldersFileName: 'folders.json', + assetsFileName: 'assets.json', + fieldsImportInvalidKeys: [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'asset_types_count', + ], + assetTypesImportInvalidKeys: [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'category', + 'preview_image_url', + 'category_detail', + ], + mapperRootDir: 'mapper', + mapperAssetsModuleDir: 'assets', + mapperUidFileName: 'uid-mapping.json', + mapperUrlFileName: 'url-mapping.json', + mapperSpaceUidFileName: 'space-uid-mapping.json', + uploadAssetsConcurrency: 2, + importFoldersConcurrency: 1, + }, 'assets-old': { dirName: 'assets', fileName: 'assets.json', diff --git a/packages/contentstack-import/src/constants/index.ts b/packages/contentstack-import/src/constants/index.ts index 5872ec29..e3455787 100644 --- a/packages/contentstack-import/src/constants/index.ts +++ b/packages/contentstack-import/src/constants/index.ts @@ -20,6 +20,7 @@ export const PATH_CONSTANTS = { INDEX: 'index.json', FOLDER_MAPPING: 'folder-mapping.json', VERSIONED_ASSETS: 'versioned-assets.json', + SPACE_UID_MAPPING: 'space-uid-mapping.json', }, /** Module subdirectory names within mapper */ diff --git a/packages/contentstack-import/src/import/modules/assets.ts b/packages/contentstack-import/src/import/modules/assets.ts index 007f2187..46e2100c 100644 --- a/packages/contentstack-import/src/import/modules/assets.ts +++ b/packages/contentstack-import/src/import/modules/assets.ts @@ -9,16 +9,13 @@ import { existsSync } from 'node:fs'; import includes from 'lodash/includes'; import { v4 as uuid } from 'uuid'; import { resolve as pResolve, join } from 'node:path'; -import { - FsUtility, - log, - handleAndLogError, -} from '@contentstack/cli-utilities'; +import { FsUtility, log, handleAndLogError } from '@contentstack/cli-utilities'; +import { ImportSpaces } from '@contentstack/cli-asset-management'; import { PATH_CONSTANTS } from '../../constants'; import config from '../../config'; import { ModuleClassParams } from '../../types'; -import { formatDate, PROCESS_NAMES, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS } from '../../utils'; +import { formatDate, fsUtil, PROCESS_NAMES, MODULE_CONTEXTS, MODULE_NAMES, PROCESS_STATUS } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; export default class ImportAssets extends BaseClass { @@ -42,22 +39,14 @@ export default class ImportAssets extends BaseClass { this.currentModuleName = MODULE_NAMES[MODULE_CONTEXTS.ASSETS]; this.assetsPath = join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ASSETS); - this.mapperDirPath = join( - this.importConfig.backupDir, - PATH_CONSTANTS.MAPPER, - PATH_CONSTANTS.MAPPER_MODULES.ASSETS, - ); + this.mapperDirPath = join(this.importConfig.backupDir, PATH_CONSTANTS.MAPPER, PATH_CONSTANTS.MAPPER_MODULES.ASSETS); this.assetUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.UID_MAPPING); this.assetUrlMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.URL_MAPPING); this.assetFolderUidMapperPath = join(this.mapperDirPath, PATH_CONSTANTS.FILES.FOLDER_MAPPING); this.assetsRootPath = join(this.importConfig.backupDir, this.assetConfig.dirName); this.fs = new FsUtility({ basePath: this.mapperDirPath }); this.environments = this.fs.readFile( - join( - this.importConfig.backupDir, - PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, - PATH_CONSTANTS.FILES.ENVIRONMENTS, - ), + join(this.importConfig.backupDir, PATH_CONSTANTS.CONTENT_DIRS.ENVIRONMENTS, PATH_CONSTANTS.FILES.ENVIRONMENTS), true, ) as Record; } @@ -70,6 +59,97 @@ export default class ImportAssets extends BaseClass { try { log.debug('Starting assets import process...', this.importConfig.context); + // AM 2.0: assetManagementEnabled is set in the config handler when spaces/ + am_v2 are detected. + if (this.importConfig.assetManagementEnabled) { + const assetManagementUrl = this.importConfig.assetManagementUrl; + if (!assetManagementUrl) { + log.info( + 'AM 2.0 export detected but assetManagementUrl is not configured in the region settings. Skipping AM 2.0 asset import.', + this.importConfig.context, + ); + return; + } + + const progress = this.createNestedProgress(this.currentModuleName); + try { + const assetManagementModuleConfig = this.importConfig.modules['asset-management']; + const importer = new ImportSpaces({ + contentDir: this.importConfig.contentDir, + assetManagementUrl, + org_uid: this.importConfig.org_uid ?? '', + apiKey: this.importConfig.apiKey, + host: this.importConfig.region?.cma ?? this.importConfig.host ?? '', + sourceApiKey: this.importConfig.source_stack, + context: this.importConfig.context as unknown as Record, + backupDir: this.importConfig.backupDir, + apiConcurrency: this.importConfig.modules?.apiConcurrency, + spacesDirName: assetManagementModuleConfig?.dirName, + fieldsDir: assetManagementModuleConfig?.fieldsDir, + assetTypesDir: assetManagementModuleConfig?.assetTypesDir, + fieldsFileName: assetManagementModuleConfig?.fieldsFileName, + assetTypesFileName: assetManagementModuleConfig?.assetTypesFileName, + foldersFileName: assetManagementModuleConfig?.foldersFileName, + assetsFileName: assetManagementModuleConfig?.assetsFileName, + fieldsImportInvalidKeys: assetManagementModuleConfig?.fieldsImportInvalidKeys, + assetTypesImportInvalidKeys: assetManagementModuleConfig?.assetTypesImportInvalidKeys, + mapperRootDir: assetManagementModuleConfig?.mapperRootDir ?? PATH_CONSTANTS.MAPPER, + mapperAssetsModuleDir: + assetManagementModuleConfig?.mapperAssetsModuleDir ?? PATH_CONSTANTS.MAPPER_MODULES.ASSETS, + mapperUidFileName: assetManagementModuleConfig?.mapperUidFileName ?? PATH_CONSTANTS.FILES.UID_MAPPING, + mapperUrlFileName: assetManagementModuleConfig?.mapperUrlFileName ?? PATH_CONSTANTS.FILES.URL_MAPPING, + mapperSpaceUidFileName: + assetManagementModuleConfig?.mapperSpaceUidFileName ?? PATH_CONSTANTS.FILES.SPACE_UID_MAPPING, + }); + importer.setParentProgressManager(progress); + + const { spaceMappings } = await importer.start(); + + // Link imported AM spaces to the target stack via CMA branch settings. + if (spaceMappings.length > 0) { + try { + const branchUid = this.importConfig.branchName ?? 'main'; + + const branchData = (await this.stack.branch(branchUid).fetch({ include_settings: true })) as Record< + string, + any + >; + const currentLinked = (branchData?.settings?.am_v2?.linked_workspaces ?? []) as Array<{ + uid: string; + space_uid: string; + is_default: boolean; + operation?: string; + }>; + + const newWorkspaces = spaceMappings.map(({ newSpaceUid, workspaceUid }) => ({ + uid: workspaceUid, + space_uid: newSpaceUid, + is_default: false, + operation: 'LINK', + })); + + const combinedWorkspaces = [...currentLinked, ...newWorkspaces]; + + await this.stack.branch(branchUid).updateSettings({ + branch: { settings: { am_v2: { linked_workspaces: combinedWorkspaces } } }, + }); + log.success( + `Linked ${newWorkspaces.length} space(s) to branch "${branchUid}"`, + this.importConfig.context, + ); + } catch (linkErr) { + handleAndLogError(linkErr, { ...this.importConfig.context }); + } + } + + this.completeProgressWithMessage(); + } catch (error) { + this.completeProgress(false, (error as Error)?.message ?? 'AM 2.0 asset import failed'); + throw error; + } + return; + } + // Legacy flow continues below + // Step 1: Analyze import data const [foldersCount, assetsCount, versionedAssetsCount, publishableAssetsCount] = await this.withLoadingSpinner( 'ASSETS: Analyzing import data...', @@ -221,9 +301,7 @@ export default class ImportAssets extends BaseClass { */ async importAssets(isVersion = false): Promise { const processName = isVersion ? 'import versioned assets' : 'import assets'; - const indexFileName = isVersion - ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS - : this.assetConfig.fileName; + const indexFileName = isVersion ? PATH_CONSTANTS.FILES.VERSIONED_ASSETS : this.assetConfig.fileName; const basePath = isVersion ? join(this.assetsPath, 'versions') : this.assetsPath; const progressProcessName = isVersion ? PROCESS_NAMES.ASSET_VERSIONS : PROCESS_NAMES.ASSET_UPLOAD; diff --git a/packages/contentstack-import/src/import/modules/stack.ts b/packages/contentstack-import/src/import/modules/stack.ts index d6d920b0..969ad4c9 100644 --- a/packages/contentstack-import/src/import/modules/stack.ts +++ b/packages/contentstack-import/src/import/modules/stack.ts @@ -1,4 +1,5 @@ import { join } from 'node:path'; +import { existsSync } from 'node:fs'; import { log, handleAndLogError } from '@contentstack/cli-utilities'; import { PATH_CONSTANTS } from '../../constants'; @@ -59,7 +60,6 @@ export default class ImportStack extends BaseClass { await this.importStackSettings(); this.completeProgressWithMessage(); - } catch (error) { this.completeProgress(false, 'Stack settings import failed'); handleAndLogError(error, { ...this.importConfig.context }); @@ -69,6 +69,17 @@ export default class ImportStack extends BaseClass { private async importStackSettings(): Promise { log.debug('Processing stack settings for import', this.importConfig.context); + // Old source-org space UIDs must not be written to the target stack — + // the asset-management module will apply the correct am_v2.linked_workspaces. + if (existsSync(join(this.importConfig.contentDir, 'spaces'))) { + const { am_v2, ...settingsWithoutAm } = this.stackSettings as any; + this.stackSettings = settingsWithoutAm; + log.debug( + 'Stripped am_v2 from stack settings; asset-management module will apply it after space creation', + this.importConfig.context, + ); + } + // Update environment UID mapping if live preview is configured if (this.stackSettings?.live_preview && this.stackSettings?.live_preview['default-env'] !== undefined) { const oldEnvUid = this.stackSettings.live_preview['default-env']; diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index 2b7c3bd9..29e85980 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -72,6 +72,24 @@ export default interface DefaultConfig { uploadAssetsConcurrency: number; importFoldersConcurrency: number; }; + 'asset-management': { + dirName: string; + fieldsDir: string; + assetTypesDir: string; + fieldsFileName: string; + assetTypesFileName: string; + foldersFileName: string; + assetsFileName: string; + fieldsImportInvalidKeys: string[]; + assetTypesImportInvalidKeys: string[]; + mapperRootDir: string; + mapperAssetsModuleDir: string; + mapperUidFileName: string; + mapperUrlFileName: string; + mapperSpaceUidFileName: string; + uploadAssetsConcurrency: number; + importFoldersConcurrency: number; + }; content_types: { dirName: string; fileName: string; diff --git a/packages/contentstack-import/src/types/import-config.ts b/packages/contentstack-import/src/types/import-config.ts index 86db5668..5025c82a 100644 --- a/packages/contentstack-import/src/types/import-config.ts +++ b/packages/contentstack-import/src/types/import-config.ts @@ -58,6 +58,8 @@ export default interface ImportConfig extends DefaultConfig, ExternalConfig { personalizeProjectName?: string; 'exclude-global-modules': false; context: Context; + assetManagementUrl?: string; + assetManagementEnabled?: boolean; } type branch = { diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 70bf9110..a73584b3 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -19,6 +19,7 @@ export interface Region { cma: string; cda: string; uiHost: string; + assetManagementUrl?: string; } export interface InquirePayload { diff --git a/packages/contentstack-import/src/utils/import-config-handler.ts b/packages/contentstack-import/src/utils/import-config-handler.ts index 0eb0ee29..927e3448 100644 --- a/packages/contentstack-import/src/utils/import-config-handler.ts +++ b/packages/contentstack-import/src/utils/import-config-handler.ts @@ -1,5 +1,6 @@ import merge from 'merge'; import * as path from 'path'; +import { existsSync, readFileSync } from 'node:fs'; import { omit, filter, includes, isArray } from 'lodash'; import { configHandler, isAuthenticated, cliux, sanitizePath, log } from '@contentstack/cli-utilities'; import defaultConfig from '../config'; @@ -21,7 +22,6 @@ const setupConfig = async (importCmdFlags: any): Promise => { if (importCmdFlags['config']) { let externalConfig = await readFile(importCmdFlags['config']); - if (isArray(externalConfig['modules'])) { config.modules.types = filter(config.modules.types, (module) => includes(externalConfig['modules'], module)); externalConfig = omit(externalConfig, ['modules']); @@ -126,6 +126,40 @@ const setupConfig = async (importCmdFlags: any): Promise => { config['exclude-global-modules'] = importCmdFlags['exclude-global-modules']; } + const spacesDir = path.join(config.contentDir, 'spaces'); + const stackSettingsPath = path.join(config.contentDir, 'stack', 'settings.json'); + + if (existsSync(spacesDir) && existsSync(stackSettingsPath)) { + try { + const stackSettings = JSON.parse(readFileSync(stackSettingsPath, 'utf8')); + if (stackSettings?.am_v2) { + config.assetManagementEnabled = true; + config.assetManagementUrl = configHandler.get('region')?.assetManagementUrl; + + const branchesJsonCandidates = [ + path.join(config.contentDir, 'branches.json'), + path.join(config.contentDir, '..', 'branches.json'), + ]; + for (const branchesJsonPath of branchesJsonCandidates) { + if (existsSync(branchesJsonPath)) { + try { + const branches = JSON.parse(readFileSync(branchesJsonPath, 'utf8')); + const apiKey = branches?.[0]?.stackHeaders?.api_key; + if (apiKey) { + config.source_stack = apiKey; + } + } catch { + // branches.json unreadable — URL mapping will be skipped + } + break; + } + } + } + } catch { + // stack settings unreadable — not an AM 2.0 export we can process + } + } + // Add authentication details to config for context tracking config.authenticationMethod = authenticationMethod; log.debug('Import configuration setup completed.', { ...config });