Feat/dx 9341

.talismanrc

@@ -1,4 +1,12 @@
 fileignoreconfig:
 - filename: pnpm-lock.yaml
   checksum: 705ce49f9667e34fb64082025a41a1e3b950c1d2f968c3d4faf9461943ae599a
-version: ""
+- filename: packages/contentstack-export/src/utils/export-config-handler.ts
+  checksum: 68351337b78f0962475498946fe34245a681182db756528c9ff6fab7153392e3
+- filename: packages/contentstack-import-setup/src/utils/import-config-handler.ts
+  checksum: c13ca4996b3767fae9d2716fe9ae7d90d98b26012e064f42ea9fdd054f684761
+- filename: packages/contentstack-import/src/utils/import-config-handler.ts
+  checksum: 999092528b445b6e9de6883150f02b56bf5050dc64129335d84246c157bfe303
+- filename: packages/contentstack-asset-management/src/utils/export-helpers.ts
+  checksum: 726d1632110ebc203ce70dc1cbe7d4b3011f56349ed371b6645d0375e7818cad
+version: ""

packages/contentstack-asset-management/src/constants/index.ts

@@ -13,15 +13,13 @@ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [
   'created_by',
   'updated_at',
   'updated_by',
-  'is_system',
   'asset_types_count',
 ] as const;
 export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [
   'created_at',
   'created_by',
   'updated_at',
   'updated_by',
-  'is_system',
   'category',
   'preview_image_url',
   'category_detail',

packages/contentstack-asset-management/src/export/asset-types.ts

@@ -13,26 +13,29 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter {
     super(apiConfig, exportContext);
   }
 
-  async start(spaceUid: string): Promise<void> {
+  async start(spaceUid: string): Promise<number> {
     await this.init();
 
     log.debug('Starting shared asset types export process...', this.exportContext.context);
+    log.info('Exporting shared asset types...', this.exportContext.context);
 
     const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid, this.apiPageSize, this.apiFetchConcurrency);
     const items = getArrayFromResponse(assetTypesData, 'asset_types');
     const dir = this.getAssetTypesDir();
-    if (items.length === 0) {
-      log.info('No asset types to export, writing empty asset-types', this.exportContext.context);
-    } else {
-      log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
-    }
     await this.writeItemsToChunkedJson(
       dir,
       'asset-types.json',
       'asset_types',
       ['uid', 'title', 'category', 'file_extension'],
       items,
     );
+    log.info(
+      items.length === 0
+        ? 'No asset types to export'
+        : `Exported ${items.length} shared asset type(s)`,
+      this.exportContext.context,
+    );
     this.tick(true, `asset_types (${items.length})`, null);
+    return items.length;
   }
 }

packages/contentstack-asset-management/src/export/assets.ts

@@ -2,12 +2,13 @@ import { resolve as pResolve } from 'node:path';
 import { Readable } from 'node:stream';
 import { mkdir, writeFile } from 'node:fs/promises';
 import chunk from 'lodash/chunk';
-import { configHandler, log, FsUtility } from '@contentstack/cli-utilities';
+import { log, FsUtility } from '@contentstack/cli-utilities';
 
 import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
 import type { ExportContext } from '../types/export-types';
 import { CSAssetsExportAdapter } from './base';
-import { writeStreamToFile } from '../utils/export-helpers';
+import { writeStreamToFile, getArrayFromResponse, getSecuredAssetAuth, SecuredAssetAuthError } from '../utils/export-helpers';
+import type { SecuredAssetAuth } from '../utils/export-helpers';
 import { forEachChunkedJsonStore } from '../utils/chunked-json-reader';
 import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry';
 import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter';
@@ -17,6 +18,9 @@ const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid'];
 
 type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string };
 
+/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */
+export type SpaceExportCounts = { assets: number; folders: number };
+
 export default class ExportAssets extends CSAssetsExportAdapter {
   constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
     super(apiConfig, exportContext);
@@ -26,7 +30,7 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     return Boolean(asset?.url && (asset?.uid ?? asset?._uid));
   }
 
-  async start(workspace: LinkedWorkspace, spaceDir: string): Promise<void> {
+  async start(workspace: LinkedWorkspace, spaceDir: string): Promise<SpaceExportCounts> {
     await this.init();
 
     log.debug(`Starting assets export for space ${workspace.space_uid}`, this.exportContext.context);
@@ -75,19 +79,21 @@ export default class ExportAssets extends CSAssetsExportAdapter {
     this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null);
 
     log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
-    await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
+    const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
+
+    const folderCount = getArrayFromResponse(folders, 'folders').length;
+    return { assets: assetsDownloaded, folders: folderCount };
   }
 
   /**
    * Download asset binaries by reading the just-written chunked `assets.json` back from disk
    * (one chunk at a time), so we never re-materialize the whole asset list in memory.
    */
-  private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<void> {
+  private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<number> {
     const filesDir = pResolve(assetsDir, 'files');
     await mkdir(filesDir, { recursive: true });
 
     const securedAssets = this.exportContext.securedAssets ?? false;
-    const authtoken = securedAssets ? configHandler.get('authtoken') : null;
     log.debug(
       `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}, expected=${expectedDownloads}`,
       this.exportContext.context,
@@ -96,6 +102,9 @@ export default class ExportAssets extends CSAssetsExportAdapter {
 
     let downloadOk = 0;
     let downloadFail = 0;
+    // Set when a 401 persists after a forced token refresh — from then on, skip the network
+    // entirely and abort the phase, instead of individually failing every remaining asset.
+    let authFailure: SecuredAssetAuthError | null = null;
 
     await forEachChunkedJsonStore<AssetRecord>(
       assetsDir,
@@ -105,29 +114,85 @@ export default class ExportAssets extends CSAssetsExportAdapter {
         chunkReadLogLabel: 'assets',
         onOpenError: (err) => log.debug(`Could not open assets.json for download: ${err}`, this.exportContext.context),
         onEmptyIndexer: () => log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context),
+        // A chunk that fails to read back would otherwise drop its downloads silently. `records` are
+        // recovered from metadata.json, so we count + surface each lost asset by identity here — no
+        // separate full-metadata reconcile (which would re-materialize the whole set every run).
+        onChunkError: (records, err) => {
+          log.error(
+            `Failed to read an asset chunk back from disk during download for space ${spaceUid}: ${
+              (err as Error)?.message ?? String(err)
+            }`,
+            this.exportContext.context,
+          );
+          for (const rec of records) {
+            if (!this.isDownloadable(rec)) continue;
+            downloadFail += 1;
+            const label = rec.file_name ?? rec.filename ?? rec.uid ?? 'asset';
+            this.tick(false, `asset: ${label}`, 'Asset chunk unreadable');
+            log.error(
+              `Asset ${rec.uid ?? '<unknown>'} not downloaded — chunk unreadable for space ${spaceUid}`,
+              this.exportContext.context,
+            );
+          }
+        },
       },
       async (records) => {
         const valid = records.filter((asset) => this.isDownloadable(asset));
         if (valid.length === 0) return;
+        const chunkAuthFailure = authFailure;
+        if (chunkAuthFailure) {
+          // Auth already failed hard — count the remaining downloadables without hitting the network.
+          for (const rec of valid) {
+            downloadFail += 1;
+            this.tick(false, `asset: ${rec.filename ?? rec.file_name ?? rec.uid ?? 'asset'}`, chunkAuthFailure.message);
+          }
+          return;
+        }
+        // Resolve per chunk so long download phases pick up proactively refreshed OAuth tokens.
+        let auth: SecuredAssetAuth = securedAssets ? await getSecuredAssetAuth() : {};
         const apiBatches = chunk(valid, this.downloadAssetsBatchConcurrency);
         const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => {
           const asset = apiBatches[batchIndex][index] as AssetRecord;
           const uid = (asset.uid ?? asset._uid) as string;
           const url = asset.url as string;
           const filename = asset.filename ?? asset.file_name ?? 'asset';
           if (!url || !uid) return;
+          const knownAuthFailure = authFailure as SecuredAssetAuthError | null;
+          if (knownAuthFailure) {
+            downloadFail += 1;
+            this.tick(false, `asset: ${filename}`, knownAuthFailure.message);
+            return;
+          }
           try {
             const separator = url.includes('?') ? '&' : '?';
-            const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url;
+            const doFetch = () =>
+              fetch(
+                securedAssets && auth.authtoken ? `${url}${separator}authtoken=${auth.authtoken}` : url,
+                securedAssets && auth.headers ? { headers: auth.headers } : undefined,
+              );
             // Binary GET is idempotent — retry transient failures with backoff.
             const response = await withRetry(
               async () => {
                 let resp: Response;
                 try {
-                  resp = await fetch(downloadUrl);
+                  resp = await doFetch();
                 } catch (e) {
                   throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`);
                 }
+                if (securedAssets && resp.status === 401) {
+                  // Token expired or was revoked mid-run — force one refresh (deduped upstream)
+                  // and refetch. A second 401 means auth is unrecoverable: abort the phase.
+                  auth = await getSecuredAssetAuth(true);
+                  try {
+                    resp = await doFetch();
+                  } catch (e) {
+                    throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`);
+                  }
+                  if (resp.status === 401) {
+                    authFailure = new SecuredAssetAuthError(resp.status);
+                    throw authFailure;
+                  }
+                }
                 if (!resp.ok) {
                   if (isRetryableStatus(resp.status)) {
                     throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after')));
@@ -153,24 +218,25 @@ export default class ExportAssets extends CSAssetsExportAdapter {
             downloadFail += 1;
             const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
             this.tick(false, `asset: ${filename}`, err);
-            log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
+            log.error(
+              `Failed to download asset ${uid} (${filename}): ${(e as Error)?.message ?? String(e)}`,
+              this.exportContext.context,
+            );
           }
         };
 
         await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler);
       },
     );
 
-    // Completeness check: a chunk that fails to read back is skipped (logged at debug) by
-    // forEachChunkedJsonStore, which would silently drop those downloads. Reconcile attempts
-    // (ok + failed) against what streaming counted as downloadable.
-    const attempted = downloadOk + downloadFail;
-    if (attempted < expectedDownloads) {
-      log.warn(
-        `Asset downloads for space ${spaceUid} incomplete: expected ${expectedDownloads}, attempted ${attempted}` +
-          ` — ${expectedDownloads - attempted} asset(s) were never read back for download.`,
+    const terminalAuthFailure = authFailure as SecuredAssetAuthError | null;
+    if (terminalAuthFailure) {
+      // Fail the space loudly — a "completed with errors" summary would bury the real cause.
+      log.error(
+        `Aborted asset downloads for space ${spaceUid}: ${terminalAuthFailure.message}`,
         this.exportContext.context,
       );
+      throw terminalAuthFailure;
     }
 
     log.info(
@@ -180,8 +246,11 @@ export default class ExportAssets extends CSAssetsExportAdapter {
       this.exportContext.context,
     );
     log.debug(
-      `Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`,
+      `Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, expected=${expectedDownloads}`,
       this.exportContext.context,
     );
+
+    return downloadOk;
   }
+
 }

packages/contentstack-asset-management/src/export/fields.ts

@@ -13,20 +13,21 @@ export default class ExportFields extends CSAssetsExportAdapter {
     super(apiConfig, exportContext);
   }
 
-  async start(spaceUid: string): Promise<void> {
+  async start(spaceUid: string): Promise<number> {
     await this.init();
 
     log.debug('Starting shared fields export process...', this.exportContext.context);
+    log.info('Exporting shared fields...', this.exportContext.context);
 
     const fieldsData = await this.getWorkspaceFields(spaceUid, this.apiPageSize, this.apiFetchConcurrency);
     const items = getArrayFromResponse(fieldsData, 'fields');
     const dir = this.getFieldsDir();
-    if (items.length === 0) {
-      log.info('No field items to export, writing empty fields', this.exportContext.context);
-    } else {
-      log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
-    }
     await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
+    log.info(
+      items.length === 0 ? 'No fields to export' : `Exported ${items.length} shared field(s)`,
+      this.exportContext.context,
+    );
     this.tick(true, `fields (${items.length})`, null);
+    return items.length;
   }
 }

packages/contentstack-asset-management/src/export/index.ts

@@ -1,4 +1,5 @@
 export { ExportSpaces, exportSpaceStructure } from './spaces';
+export type { AssetExportCounts } from './spaces';
 export { default as ExportAssetTypes } from './asset-types';
 export { default as ExportFields } from './fields';
 export { default as ExportAssets } from './assets';

packages/contentstack-asset-management/src/export/spaces.ts

@@ -9,6 +9,17 @@ import ExportAssetTypes from './asset-types';
 import ExportFields from './fields';
 import ExportWorkspace from './workspaces';
 
+/**
+ * Real entity counts for the export summary (Bug 3 — "everything under ASSETS"):
+ * assets = downloaded binaries, folders = folder entities, plus shared asset_types and fields.
+ */
+export type AssetExportCounts = {
+  assets: number;
+  folders: number;
+  assetTypes: number;
+  fields: number;
+};
+
 /**
  * Orchestrates the full Contentstack Assets export: shared asset types and fields,
  * then per-workspace metadata and assets (including internal download).
@@ -27,7 +38,7 @@ export class ExportSpaces {
     this.parentProgressManager = parent;
   }
 
-  async start(): Promise<void> {
+  async start(): Promise<AssetExportCounts> {
     const {
       linkedWorkspaces,
       exportDir,
@@ -42,7 +53,7 @@ export class ExportSpaces {
 
     if (!linkedWorkspaces.length) {
       log.debug('No linked workspaces to export', context);
-      return;
+      return { assets: 0, folders: 0, assetTypes: 0, fields: 0 };
     }
 
     log.debug('Starting Contentstack Assets export process...', context);
@@ -91,6 +102,11 @@ export class ExportSpaces {
     const firstSpaceUid = linkedWorkspaces[0].space_uid;
     let bootstrapFailed = false;
     let anySpaceFailed = false;
+    // Real entity counts accumulated for the summary (Bug 3).
+    let assetsTotal = 0;
+    let foldersTotal = 0;
+    let assetTypesCount = 0;
+    let fieldsCount = 0;
     try {
       progress.startProcess(PROCESS_NAMES.AM_FIELDS);
       progress.startProcess(PROCESS_NAMES.AM_ASSET_TYPES);
@@ -100,7 +116,10 @@ export class ExportSpaces {
       const exportFields = new ExportFields(apiConfig, exportContext);
       exportFields.setParentProgressManager(progress);
       try {
-        await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
+        [assetTypesCount, fieldsCount] = await Promise.all([
+          exportAssetTypes.start(firstSpaceUid),
+          exportFields.start(firstSpaceUid),
+        ]);
         progress.completeProcess(PROCESS_NAMES.AM_FIELDS, true);
         progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, true);
       } catch (bootstrapErr) {
@@ -118,7 +137,9 @@ export class ExportSpaces {
         try {
           const exportWorkspace = new ExportWorkspace(apiConfig, exportContext);
           exportWorkspace.setParentProgressManager(progress);
-          await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
+          const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
+          assetsTotal += spaceCounts.assets;
+          foldersTotal += spaceCounts.folders;
           progress.completeProcess(spaceProcess, true);
           log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
         } catch (err) {
@@ -142,6 +163,8 @@ export class ExportSpaces {
         context,
       );
       log.debug('Contentstack Assets export completed', context);
+
+      return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount };
     } catch (err) {
       if (!bootstrapFailed) {
         // Mark any spaces that hadn't been processed as failed so the multibar
@@ -170,6 +193,6 @@ export class ExportSpaces {
 /**
  * Entry point for callers that prefer a function. Delegates to ExportSpaces.
  */
-export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise<void> {
-  await new ExportSpaces(options).start();
+export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise<AssetExportCounts> {
+  return new ExportSpaces(options).start();
 }