From 7816c7c08ec82f4c291c66da6f6a260465549ac6 Mon Sep 17 00:00:00 2001 From: yashin4112 Date: Mon, 6 Jul 2026 13:39:07 +0530 Subject: [PATCH 1/2] feat(assets): add extractAssets function for asset mapping across Contentful, Drupal, and WordPress --- upload-api/migration-contentful/index.js | 4 +- .../libs/extractAssets.js | 98 +++++++++++++++++++ upload-api/migration-drupal/index.js | 4 +- .../migration-drupal/libs/extractAssets.js | 94 ++++++++++++++++++ upload-api/migration-wordpress/index.ts | 4 +- .../migration-wordpress/libs/extractAssets.ts | 97 ++++++++++++++++++ upload-api/src/controllers/wordpress/index.ts | 5 +- upload-api/src/services/contentful/index.ts | 9 +- upload-api/src/services/drupal/index.ts | 8 +- .../controllers/wordpress.controller.test.ts | 5 +- 10 files changed, 318 insertions(+), 10 deletions(-) create mode 100644 upload-api/migration-contentful/libs/extractAssets.js create mode 100644 upload-api/migration-drupal/libs/extractAssets.js create mode 100644 upload-api/migration-wordpress/libs/extractAssets.ts diff --git a/upload-api/migration-contentful/index.js b/upload-api/migration-contentful/index.js index babec0d9b..304304296 100644 --- a/upload-api/migration-contentful/index.js +++ b/upload-api/migration-contentful/index.js @@ -5,11 +5,13 @@ const createInitialMapper = require('./libs/createInitialMapper'); const extractLocale = require('./libs/extractLocale'); const extractTaxonomy = require('./libs/extractTaxonomy'); const extractEntries = require('./libs/extractEntries'); +const extractAssets = require('./libs/extractAssets'); module.exports = { extractContentTypes, createInitialMapper, extractLocale, extractTaxonomy, - extractEntries + extractEntries, + extractAssets }; diff --git a/upload-api/migration-contentful/libs/extractAssets.js b/upload-api/migration-contentful/libs/extractAssets.js new file mode 100644 index 000000000..e44cdb33b --- /dev/null +++ b/upload-api/migration-contentful/libs/extractAssets.js @@ -0,0 +1,98 @@ +'use strict'; +/* eslint-disable @typescript-eslint/no-var-requires */ + +const { readFile } = require('../utils/helper'); + +/** + * Extracts asset mapping rows from a Contentful export. + * + * Discovery mirrors extractEntries in this same package: the Contentful export + * is a single JSON file read via readFile(cleanLocalPath), with top-level + * `entries`, `locales` and `assets` arrays (standard `contentful-export` + * output). Each asset carries a stable `sys.id` and locale-keyed fields, so + * file metadata lives under `fields.file[locale]` and the title under + * `fields.title[locale]`: + * + * { + * sys: { id: '' }, + * fields: { + * title: { 'en-US': 'My Asset' }, + * file: { + * 'en-US': { + * fileName: 'my-asset.png', + * url: '//images.ctfassets.net/.../my-asset.png', + * details: { size: 12345 } + * } + * } + * } + * } + * + * We dedupe by sys.id and use it as both `id` and `otherCmsAssetUid` so the row + * stays stable across delta iterations. Returned rows are consumed by + * putTestData on the main API to populate the asset_mapper DB (AssetMapper UI), + * matching the AEM/Sitecore AssetMappingRow shape. + * + * @param {string} cleanLocalPath - Path to the Contentful export JSON file. + * @returns {Array<{id:string,otherCmsAssetUid:string,filename:string,title:string,file_size:(number|string),assetPath:string,isUpdate:boolean}>} + */ +const extractAssets = (cleanLocalPath) => { + try { + const alldata = readFile(cleanLocalPath); + const assets = alldata?.assets; + const locales = alldata?.locales?.map((locale) => locale?.code) ?? []; + + if (!assets || !Array.isArray(assets) || assets.length === 0) { + console.info('No assets found in Contentful export'); + return []; + } + + const rows = []; + const seenIds = new Set(); + + // Prefer a locale-keyed value, falling back to the first available locale. + const pickLocalized = (field) => { + if (!field || typeof field !== 'object') return undefined; + for (const locale of locales) { + if (field[locale] !== undefined) return field[locale]; + } + const firstKey = Object.keys(field)[0]; + return firstKey !== undefined ? field[firstKey] : undefined; + }; + + for (const asset of assets) { + const id = asset?.sys?.id; + if (!id || seenIds.has(id)) { + continue; + } + + const file = pickLocalized(asset?.fields?.file); + const title = pickLocalized(asset?.fields?.title); + + const filename = file?.fileName ?? title ?? ''; + const fileSize = file?.details?.size ?? ''; + // Contentful serves assets from a CDN URL rather than a folder path. + const assetPath = file?.url ?? ''; + + seenIds.add(id); + + rows.push({ + id, + otherCmsAssetUid: id, + filename, + title: title ?? filename, + file_size: fileSize, + assetPath, + isUpdate: false, + }); + } + + console.info(`extractAssets: Extracted ${rows.length} Contentful assets`); + + return rows; + } catch (err) { + console.error('Error extracting Contentful assets:', err); + return []; + } +}; + +module.exports = extractAssets; \ No newline at end of file diff --git a/upload-api/migration-drupal/index.js b/upload-api/migration-drupal/index.js index ec551137e..c46e7049b 100644 --- a/upload-api/migration-drupal/index.js +++ b/upload-api/migration-drupal/index.js @@ -4,11 +4,13 @@ const extractTaxonomy = require('./libs/extractTaxonomy'); const createInitialMapper = require('./libs/createInitialMapper'); const extractLocale = require('./libs/extractLocale'); const extractEntries = require('./libs/extractEntries'); +const extractAssets = require('./libs/extractAssets'); module.exports = { // extractContentTypes, extractTaxonomy, createInitialMapper, extractLocale, - extractEntries + extractEntries, + extractAssets }; diff --git a/upload-api/migration-drupal/libs/extractAssets.js b/upload-api/migration-drupal/libs/extractAssets.js new file mode 100644 index 000000000..7b49f3370 --- /dev/null +++ b/upload-api/migration-drupal/libs/extractAssets.js @@ -0,0 +1,94 @@ +'use strict'; +/* eslint-disable @typescript-eslint/no-var-requires */ + +/** + * Builds the assetMapping array consumed by putTestData on the main API to + * populate the asset_mapper DB shown on the AssetMapper UI — same flow/shape + * as AEM/Sitecore/Contentful/WordPress. + * + * Drupal reads from MySQL (not files). Files live in the `file_managed` table + * (fid, uuid, filename, uri, filesize, filemime). We surface every managed + * file, deduped by its Drupal file id, using the fid as the stable + * id/otherCmsAssetUid so it lines up across delta iterations. + * + * Row shape mirrors migration-sitecore/libs/extractAssets.js: + * { id, otherCmsAssetUid, filename, title, file_size, assetPath, isUpdate } + */ + +const { dbConnection } = require('../utils/helper'); + +/** + * Derive a display-friendly path from the Drupal file uri. + * e.g. "public://2023-01/photo.jpg" -> "public/2023-01" + */ +const assetPathFromUri = (uri, filename) => { + if (!uri || typeof uri !== 'string') { + return ''; + } + // Strip the stream wrapper scheme ("public://", "private://", "s3://", ...) + let cleaned = uri.replace(/^[a-z0-9]+:\/\//i, ''); + // Drop the filename to leave only the directory portion + if (filename && cleaned.endsWith(filename)) { + cleaned = cleaned.slice(0, cleaned.length - filename.length); + } + return cleaned.replace(/\/+$/, ''); +}; + +/** + * @param {Object} config - Migration config; must contain config.mysql (DB + * connection details). config.assetsConfig is accepted but unused for now. + * @returns {Promise>} assetMapping rows + */ +const extractAssets = async (config) => { + const rows = []; + let connection; + + try { + connection = dbConnection(config); + + const query = ` + SELECT fid, uuid, filename, uri, filesize, filemime + FROM file_managed + ORDER BY fid + `; + const [results] = await connection.promise().query(query); + + const seenIds = new Set(); + + for (const file of results || []) { + if (!file) { + continue; + } + + const id = file.fid != null ? String(file.fid) : ''; + if (!id || seenIds.has(id)) { + continue; + } + seenIds.add(id); + + const filename = file.filename ? String(file.filename) : `File ${id}`; + + rows.push({ + id, + otherCmsAssetUid: id, + filename, + title: filename, + file_size: file.filesize != null ? String(file.filesize) : '', + assetPath: assetPathFromUri(file.uri, file.filename), + isUpdate: false + }); + } + + console.info(`extractAssets (Drupal): ${rows.length} asset(s)`); + return rows; + } catch (err) { + console.error('extractAssets (Drupal) error:', err?.message || err); + return rows; + } finally { + if (connection) { + connection.end(); + } + } +}; + +module.exports = extractAssets; \ No newline at end of file diff --git a/upload-api/migration-wordpress/index.ts b/upload-api/migration-wordpress/index.ts index 7b4b9eaf7..86192d64f 100644 --- a/upload-api/migration-wordpress/index.ts +++ b/upload-api/migration-wordpress/index.ts @@ -2,10 +2,12 @@ import extractContentTypes from './libs/contentTypes'; //import contentTypeMaker from './libs/contentTypeMapper'; import extractLocale from './libs/extractLocale'; import extractEntries from './libs/extractEntries'; +import extractAssets from './libs/extractAssets'; export { extractContentTypes, //contentTypeMaker, extractLocale, - extractEntries + extractEntries, + extractAssets } \ No newline at end of file diff --git a/upload-api/migration-wordpress/libs/extractAssets.ts b/upload-api/migration-wordpress/libs/extractAssets.ts new file mode 100644 index 000000000..35bfd0e46 --- /dev/null +++ b/upload-api/migration-wordpress/libs/extractAssets.ts @@ -0,0 +1,97 @@ +import fs from 'fs'; + +export interface AssetMappingRow { + id: string; + otherCmsAssetUid: string; + filename: string; + title: string; + file_size: number | string; + assetPath: string; + isUpdate: boolean; +} + +const normalizeArray = (value: T | T[] | undefined): T[] => { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +}; + +/** + * WordPress WXR exports are parsed upstream into JSON before reaching here, so + * we load the file exactly like extractEntries/extractItems do: read the file + * and JSON.parse it, then walk rss.channel.item. Media are `item` elements with + * `wp:post_type === 'attachment'`. + */ +const getAssetId = (item: any): string => { + const postId = item?.['wp:post_id']; + if (postId != null && String(postId).trim() !== '') { + return String(postId).trim(); + } + const guid = item?.guid?.text ?? item?.guid; + return guid != null ? String(guid).trim() : ''; +}; + +const getAssetUrl = (item: any): string => { + const url = item?.['wp:attachment_url'] ?? item?.guid?.text ?? item?.guid; + return url != null ? String(url).trim() : ''; +}; + +const getFilename = (item: any, assetUrl: string): string => { + const fromUrl = assetUrl?.split?.('/')?.pop?.(); + if (typeof fromUrl === 'string' && fromUrl.trim() !== '') { + return fromUrl.trim(); + } + const title = item?.title?.text ?? item?.title; + return typeof title === 'string' ? title.trim() : ''; +}; + +const getTitle = (item: any, filename: string): string => { + const title = item?.title?.text ?? item?.title; + if (typeof title === 'string' && title.trim() !== '') { + return title.trim(); + } + return filename.split('.').slice(0, -1).join('.') || filename; +}; + +const extractAssets = async (filePath: string): Promise => { + const rows: AssetMappingRow[] = []; + try { + const rawData = await fs.promises.readFile(filePath, 'utf8'); + const jsonData = JSON.parse(rawData); + const items = normalizeArray(jsonData?.rss?.channel?.item); + + const seenIds = new Set(); + + for (const item of items) { + if (item?.['wp:post_type'] !== 'attachment') { + continue; + } + + const id = getAssetId(item); + if (!id || seenIds.has(id)) { + continue; + } + seenIds.add(id); + + const assetPath = getAssetUrl(item); + const filename = getFilename(item, assetPath); + const title = getTitle(item, filename); + + rows.push({ + id, + otherCmsAssetUid: id, + filename, + title, + file_size: '', + assetPath, + isUpdate: false, + }); + } + + return rows; + } catch (error: any) { + console.error('Error while extracting WordPress assets:', error?.message || error); + return rows; + } +}; + +export default extractAssets; \ No newline at end of file diff --git a/upload-api/src/controllers/wordpress/index.ts b/upload-api/src/controllers/wordpress/index.ts index bf86c1361..73a1f4465 100644 --- a/upload-api/src/controllers/wordpress/index.ts +++ b/upload-api/src/controllers/wordpress/index.ts @@ -2,7 +2,7 @@ import axios from "axios"; import logger from "../../utils/logger"; import { HTTP_CODES, HTTP_TEXTS, MIGRATION_DATA_CONFIG } from "../../constants"; // eslint-disable-next-line @typescript-eslint/no-var-requires -import { extractContentTypes, extractLocale, extractEntries } from 'migration-wordpress'; +import { extractContentTypes, extractLocale, extractEntries, extractAssets } from 'migration-wordpress'; import { deleteFolderSync } from "../../helper"; import path from "path"; @@ -44,7 +44,8 @@ const createWordpressMapper = async (filePath: string = "", projectId: string | } if(contentTypeData){ - const fieldMapping: any = { contentTypes: [], extractPath: filePath }; + const assetMapping = await extractAssets(filePath); + const fieldMapping: any = { contentTypes: [], extractPath: filePath, assetMapping }; contentTypeData.forEach((contentType: any) => { const jsonfileContent = contentType; jsonfileContent.type = "content_type"; diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index 0f02c2338..acecef731 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -11,7 +11,8 @@ const { extractContentTypes, createInitialMapper, extractLocale, - extractTaxonomy + extractTaxonomy, + extractAssets } = require('migration-contentful'); const createContentfulMapper = async ( @@ -51,6 +52,9 @@ const createContentfulMapper = async ( // Must run after createInitialMapper: that step deletes contentfulMigrationData (contentfulSchema) and would remove taxonomy files written earlier. await extractTaxonomy(cleanLocalPath); + // Asset mapping rows for the AssetMapper UI (same flow as AEM/Sitecore). + const assetMapping = await extractAssets(cleanLocalPath); + let taxonomies: any[] = []; try { const taxonomyPath = path.join( @@ -77,7 +81,8 @@ const createContentfulMapper = async ( }, data: JSON.stringify({ ...initialMapper, - taxonomies + taxonomies, + assetMapping }) }; const { data} = await axios.request(req); diff --git a/upload-api/src/services/drupal/index.ts b/upload-api/src/services/drupal/index.ts index 63b33ad0f..a21006275 100644 --- a/upload-api/src/services/drupal/index.ts +++ b/upload-api/src/services/drupal/index.ts @@ -7,7 +7,7 @@ import logger from '../../utils/logger'; import { HTTP_CODES, HTTP_TEXTS } from '../../constants'; import { Config } from '../../models/types'; -const { createInitialMapper, extractLocale, extractTaxonomy } = require('migration-drupal'); +const { createInitialMapper, extractLocale, extractTaxonomy, extractAssets } = require('migration-drupal'); const createDrupalMapper = async ( config: Config, @@ -56,6 +56,9 @@ const createDrupalMapper = async ( const initialMapper = await createInitialMapper(config, affix); + // Asset mapping rows for the AssetMapper UI (same flow as AEM/Sitecore/Contentful/WordPress). + const assetMapping = await extractAssets(config); + // Read extracted taxonomies from file let taxonomies: any[] = []; try { @@ -84,7 +87,8 @@ const createDrupalMapper = async ( contentTypes: initialMapper.contentTypes, // All content types (no profile) assetsConfig: config.assetsConfig, mySQLDetails: config.mysql, - taxonomies: taxonomies // Add taxonomies to payload + taxonomies: taxonomies, // Add taxonomies to payload + assetMapping // Asset rows for the AssetMapper UI }; const req = { diff --git a/upload-api/tests/unit/controllers/wordpress.controller.test.ts b/upload-api/tests/unit/controllers/wordpress.controller.test.ts index c15351e25..e3e8aef5d 100644 --- a/upload-api/tests/unit/controllers/wordpress.controller.test.ts +++ b/upload-api/tests/unit/controllers/wordpress.controller.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const { mockAxiosRequest, mockDeleteFolderSync, mockExtractLocale, mockExtractContentTypes, mockExtractEntries } = vi.hoisted(() => ({ +const { mockAxiosRequest, mockDeleteFolderSync, mockExtractLocale, mockExtractContentTypes, mockExtractEntries, mockExtractAssets } = vi.hoisted(() => ({ mockAxiosRequest: vi.fn(), mockDeleteFolderSync: vi.fn(), mockExtractLocale: vi.fn().mockResolvedValue([]), @@ -8,12 +8,15 @@ const { mockAxiosRequest, mockDeleteFolderSync, mockExtractLocale, mockExtractCo // `extractEntries` is called between extractContentTypes and the POST to enrich each // content type with its entry list. Default: pass the input through unchanged. mockExtractEntries: vi.fn().mockImplementation((_p: string, ct: any) => ct), + // `extractAssets` builds the assetMapping rows sent with the createDummyData payload. + mockExtractAssets: vi.fn().mockResolvedValue([]), })); vi.mock('migration-wordpress', () => ({ extractLocale: mockExtractLocale, extractContentTypes: mockExtractContentTypes, extractEntries: mockExtractEntries, + extractAssets: mockExtractAssets, })); vi.mock('axios', () => ({ default: { request: mockAxiosRequest } })); From e4e3cead4d51b9d5c6c174a1649b3b127a1629b7 Mon Sep 17 00:00:00 2001 From: yashin4112 Date: Tue, 7 Jul 2026 11:55:46 +0530 Subject: [PATCH 2/2] comments resolved: enhance asset extraction and mapping for Contentful and Drupal; improve error handling and mock implementations --- .../libs/extractAssets.js | 22 +++++++---- .../migration-drupal/libs/extractAssets.js | 27 ++++++++++--- upload-api/src/services/contentful/index.ts | 10 ++--- upload-api/src/services/drupal/index.ts | 4 +- .../tests/__mocks__/migration-contentful.ts | 2 + .../tests/__mocks__/migration-drupal.ts | 1 + .../unit/services/contentful.service.test.ts | 38 ++++++++++++++++++- .../unit/services/drupal.service.test.ts | 35 ++++++++++++++++- 8 files changed, 117 insertions(+), 22 deletions(-) diff --git a/upload-api/migration-contentful/libs/extractAssets.js b/upload-api/migration-contentful/libs/extractAssets.js index e44cdb33b..a77c8f5b3 100644 --- a/upload-api/migration-contentful/libs/extractAssets.js +++ b/upload-api/migration-contentful/libs/extractAssets.js @@ -39,9 +39,11 @@ const extractAssets = (cleanLocalPath) => { try { const alldata = readFile(cleanLocalPath); const assets = alldata?.assets; - const locales = alldata?.locales?.map((locale) => locale?.code) ?? []; + const locales = Array.isArray(alldata?.locales) + ? alldata?.locales?.map((locale) => locale?.code).filter(Boolean) + : []; - if (!assets || !Array.isArray(assets) || assets.length === 0) { + if (!assets || !Array?.isArray(assets) || assets?.length === 0) { console.info('No assets found in Contentful export'); return []; } @@ -66,12 +68,18 @@ const extractAssets = (cleanLocalPath) => { } const file = pickLocalized(asset?.fields?.file); - const title = pickLocalized(asset?.fields?.title); + const titleValue = pickLocalized(asset?.fields?.title); + const title = typeof titleValue === 'string' ? titleValue : ''; - const filename = file?.fileName ?? title ?? ''; + const filename = + (typeof file?.fileName === 'string' && file?.fileName) || title || ''; const fileSize = file?.details?.size ?? ''; - // Contentful serves assets from a CDN URL rather than a folder path. - const assetPath = file?.url ?? ''; + // Contentful serves assets from a protocol-relative CDN URL ("//..."); + // normalize to https so downstream consumers get an absolute URL. + let assetPath = typeof file?.url === 'string' ? file?.url : ''; + if (assetPath.startsWith('//')) { + assetPath = `https:${assetPath}`; + } seenIds.add(id); @@ -79,7 +87,7 @@ const extractAssets = (cleanLocalPath) => { id, otherCmsAssetUid: id, filename, - title: title ?? filename, + title: title || filename, file_size: fileSize, assetPath, isUpdate: false, diff --git a/upload-api/migration-drupal/libs/extractAssets.js b/upload-api/migration-drupal/libs/extractAssets.js index 7b49f3370..d6878028b 100644 --- a/upload-api/migration-drupal/libs/extractAssets.js +++ b/upload-api/migration-drupal/libs/extractAssets.js @@ -46,12 +46,23 @@ const extractAssets = async (config) => { try { connection = dbConnection(config); + // Bound the result set so a very large media library can't be pulled fully into + // memory. Configurable via assetsConfig.maxAssets; parameterized to keep it out of + // the SQL string. + const maxAssets = Number(config?.assetsConfig?.maxAssets) || 100000; const query = ` SELECT fid, uuid, filename, uri, filesize, filemime FROM file_managed ORDER BY fid + LIMIT ? `; - const [results] = await connection.promise().query(query); + const [results] = await connection.promise().query(query, [maxAssets]); + + if (Array.isArray(results) && results.length === maxAssets) { + console.warn( + `extractAssets (Drupal): hit maxAssets cap of ${maxAssets}; some assets may be omitted` + ); + } const seenIds = new Set(); @@ -60,21 +71,21 @@ const extractAssets = async (config) => { continue; } - const id = file.fid != null ? String(file.fid) : ''; + const id = file?.fid != null ? String(file?.fid) : ''; if (!id || seenIds.has(id)) { continue; } seenIds.add(id); - const filename = file.filename ? String(file.filename) : `File ${id}`; + const filename = file?.filename ? String(file?.filename) : `File ${id}`; rows.push({ id, otherCmsAssetUid: id, filename, title: filename, - file_size: file.filesize != null ? String(file.filesize) : '', - assetPath: assetPathFromUri(file.uri, file.filename), + file_size: file?.filesize != null ? String(file?.filesize) : '', + assetPath: assetPathFromUri(file?.uri, filename), isUpdate: false }); } @@ -86,7 +97,11 @@ const extractAssets = async (config) => { return rows; } finally { if (connection) { - connection.end(); + try { + connection.end(); + } catch (endErr) { + console.error('extractAssets (Drupal) connection close error:', endErr?.message || endErr); + } } } }; diff --git a/upload-api/src/services/contentful/index.ts b/upload-api/src/services/contentful/index.ts index acecef731..81e57a23d 100644 --- a/upload-api/src/services/contentful/index.ts +++ b/upload-api/src/services/contentful/index.ts @@ -7,13 +7,13 @@ import logger from '../../utils/logger'; import { HTTP_CODES, HTTP_TEXTS } from '../../constants'; import { Config } from '../../models/types'; -const { +import { extractContentTypes, createInitialMapper, extractLocale, extractTaxonomy, extractAssets -} = require('migration-contentful'); +} from 'migration-contentful'; const createContentfulMapper = async ( projectId: string | string[], @@ -24,7 +24,7 @@ const createContentfulMapper = async ( try { const { localPath } = config; const cleanLocalPath = localPath?.replace?.(/\/$/, ''); - const fetchedLocales: [] = await extractLocale(cleanLocalPath); + const fetchedLocales: string[] = await extractLocale(cleanLocalPath); const mapperConfig = { method: 'post', @@ -47,8 +47,8 @@ const createContentfulMapper = async ( }); } - await extractContentTypes(cleanLocalPath, affix); - const initialMapper = await createInitialMapper(cleanLocalPath, affix); + await extractContentTypes(cleanLocalPath, affix as string); + const initialMapper = await createInitialMapper(cleanLocalPath, affix as string); // Must run after createInitialMapper: that step deletes contentfulMigrationData (contentfulSchema) and would remove taxonomy files written earlier. await extractTaxonomy(cleanLocalPath); diff --git a/upload-api/src/services/drupal/index.ts b/upload-api/src/services/drupal/index.ts index a21006275..ec95bf4da 100644 --- a/upload-api/src/services/drupal/index.ts +++ b/upload-api/src/services/drupal/index.ts @@ -7,7 +7,7 @@ import logger from '../../utils/logger'; import { HTTP_CODES, HTTP_TEXTS } from '../../constants'; import { Config } from '../../models/types'; -const { createInitialMapper, extractLocale, extractTaxonomy, extractAssets } = require('migration-drupal'); +import { createInitialMapper, extractLocale, extractTaxonomy, extractAssets } from 'migration-drupal'; const createDrupalMapper = async ( config: Config, @@ -52,7 +52,7 @@ const createDrupalMapper = async ( } // Extract taxonomy vocabularies and save to drupalMigrationData - await extractTaxonomy(config?.mysql); + await extractTaxonomy(config?.mysql as object); const initialMapper = await createInitialMapper(config, affix); diff --git a/upload-api/tests/__mocks__/migration-contentful.ts b/upload-api/tests/__mocks__/migration-contentful.ts index 21764d913..02ec5690f 100644 --- a/upload-api/tests/__mocks__/migration-contentful.ts +++ b/upload-api/tests/__mocks__/migration-contentful.ts @@ -3,3 +3,5 @@ import { vi } from 'vitest'; export const extractLocale = vi.fn().mockResolvedValue([]); export const extractContentTypes = vi.fn().mockResolvedValue(undefined); export const createInitialMapper = vi.fn().mockResolvedValue({ contentTypes: [] }); +export const extractTaxonomy = vi.fn().mockResolvedValue(undefined); +export const extractAssets = vi.fn().mockReturnValue([]); diff --git a/upload-api/tests/__mocks__/migration-drupal.ts b/upload-api/tests/__mocks__/migration-drupal.ts index 386318efd..4df9c9204 100644 --- a/upload-api/tests/__mocks__/migration-drupal.ts +++ b/upload-api/tests/__mocks__/migration-drupal.ts @@ -3,3 +3,4 @@ import { vi } from 'vitest'; export const extractLocale = vi.fn().mockResolvedValue(new Set()); export const extractTaxonomy = vi.fn().mockResolvedValue(undefined); export const createInitialMapper = vi.fn().mockResolvedValue({ contentTypes: [] }); +export const extractAssets = vi.fn().mockResolvedValue([]); diff --git a/upload-api/tests/unit/services/contentful.service.test.ts b/upload-api/tests/unit/services/contentful.service.test.ts index 8c1aa57ec..66af0546b 100644 --- a/upload-api/tests/unit/services/contentful.service.test.ts +++ b/upload-api/tests/unit/services/contentful.service.test.ts @@ -1,13 +1,27 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const { mockAxiosRequest } = vi.hoisted(() => ({ +const { mockAxiosRequest, mockExtractLocale, mockExtractContentTypes, mockCreateInitialMapper, mockExtractTaxonomy, mockExtractAssets } = vi.hoisted(() => ({ mockAxiosRequest: vi.fn(), + mockExtractLocale: vi.fn().mockResolvedValue([]), + mockExtractContentTypes: vi.fn().mockResolvedValue(undefined), + mockCreateInitialMapper: vi.fn().mockResolvedValue({ contentTypes: [] }), + mockExtractTaxonomy: vi.fn().mockResolvedValue(undefined), + mockExtractAssets: vi.fn().mockReturnValue([]), })); +// Mock the connector explicitly (the service uses require('migration-contentful')). +vi.mock('migration-contentful', () => ({ + extractLocale: mockExtractLocale, + extractContentTypes: mockExtractContentTypes, + createInitialMapper: mockCreateInitialMapper, + extractTaxonomy: mockExtractTaxonomy, + extractAssets: mockExtractAssets, +})); vi.mock('axios', () => ({ default: { request: mockAxiosRequest } })); vi.mock('../../../src/utils/logger', () => ({ default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() } })); import createContentfulMapper from '../../../src/services/contentful/index'; +const extractAssets = mockExtractAssets; describe('createContentfulMapper', () => { beforeEach(() => { @@ -46,4 +60,26 @@ describe('createContentfulMapper', () => { createContentfulMapper('proj-1', 'token', 'csm', config) ).resolves.toBeUndefined(); }); + + it('sends the extracted assetMapping in the createDummyData payload', async () => { + const config: any = { localPath: '/tmp/export.json' }; + const assetRows = [ + { id: 'a1', otherCmsAssetUid: 'a1', filename: 'a.png', title: 'a', file_size: 1, assetPath: 'https://x/a.png', isUpdate: false }, + ]; + (extractAssets as any).mockReturnValueOnce(assetRows); + // localeMapper POST first, createDummyData POST second. + mockAxiosRequest + .mockResolvedValueOnce({ status: 200, data: {} }) + .mockResolvedValueOnce({ data: { data: { content_mapper: [1] } } }); + + await createContentfulMapper('proj-1', 'token', 'csm', config); + + expect(extractAssets).toHaveBeenCalledWith('/tmp/export.json'); + const dummyDataCall = mockAxiosRequest.mock.calls.find( + ([req]) => typeof req?.url === 'string' && req.url.includes('createDummyData') + ); + expect(dummyDataCall).toBeTruthy(); + const payload = JSON.parse((dummyDataCall as any[])[0].data as string); + expect(payload.assetMapping).toEqual(assetRows); + }); }); diff --git a/upload-api/tests/unit/services/drupal.service.test.ts b/upload-api/tests/unit/services/drupal.service.test.ts index 774ddeccf..69edd799b 100644 --- a/upload-api/tests/unit/services/drupal.service.test.ts +++ b/upload-api/tests/unit/services/drupal.service.test.ts @@ -1,9 +1,20 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; -const { mockAxiosRequest } = vi.hoisted(() => ({ +const { mockAxiosRequest, mockExtractLocale, mockExtractTaxonomy, mockCreateInitialMapper, mockExtractAssets } = vi.hoisted(() => ({ mockAxiosRequest: vi.fn(), + mockExtractLocale: vi.fn().mockResolvedValue(new Set()), + mockExtractTaxonomy: vi.fn().mockResolvedValue(undefined), + mockCreateInitialMapper: vi.fn().mockResolvedValue({ contentTypes: [] }), + mockExtractAssets: vi.fn().mockResolvedValue([]), })); +// Mock the connector explicitly (the service imports from 'migration-drupal'). +vi.mock('migration-drupal', () => ({ + extractLocale: mockExtractLocale, + extractTaxonomy: mockExtractTaxonomy, + createInitialMapper: mockCreateInitialMapper, + extractAssets: mockExtractAssets, +})); vi.mock('axios', () => ({ default: { request: mockAxiosRequest } })); vi.mock('../../../src/utils/logger', () => ({ default: { info: vi.fn(), error: vi.fn(), warn: vi.fn(), debug: vi.fn() } })); vi.mock('fs', () => ({ @@ -13,6 +24,7 @@ vi.mock('fs', () => ({ })); import createDrupalMapper from '../../../src/services/drupal/index'; +const extractAssets = mockExtractAssets; describe('createDrupalMapper', () => { const config: any = { @@ -54,4 +66,25 @@ describe('createDrupalMapper', () => { createDrupalMapper(config, 'proj-1', 'token', 'csm') ).resolves.toBeUndefined(); }); + + it('sends the extracted assetMapping in the createDummyData payload', async () => { + const assetRows = [ + { id: '5', otherCmsAssetUid: '5', filename: 'a.jpg', title: 'a.jpg', file_size: '10', assetPath: '2023', isUpdate: false }, + ]; + (extractAssets as any).mockResolvedValueOnce(assetRows); + // localeMapper POST first, createDummyData POST second. + mockAxiosRequest + .mockResolvedValueOnce({ status: 200, data: {} }) + .mockResolvedValueOnce({ data: { data: { content_mapper: [1] } } }); + + await createDrupalMapper(config, 'proj-1', 'token', 'csm'); + + expect(extractAssets).toHaveBeenCalledWith(config); + const dummyDataCall = mockAxiosRequest.mock.calls.find( + ([req]) => typeof req?.url === 'string' && req.url.includes('createDummyData') + ); + expect(dummyDataCall).toBeTruthy(); + const payload = JSON.parse((dummyDataCall as any[])[0].data as string); + expect(payload.assetMapping).toEqual(assetRows); + }); });