From 4e39de1364ab3c1d3535d207ea5eb9e18e2ccbd5 Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 18 May 2026 12:28:55 +0530 Subject: [PATCH 1/5] added assets support in export-query --- .talismanrc | 4 + .../src/index.ts | 1 + .../query-export/am-asset-query-exporter.ts | 242 ++++++++++++++++++ .../src/query-export/index.ts | 1 + .../src/types/cs-assets-api.ts | 40 +++ .../src/utils/cs-assets-api-adapter.ts | 54 ++++ .../am-asset-query-exporter.test.ts | 152 +++++++++++ .../contentstack-query-export/package.json | 1 + .../src/commands/cm/stacks/export-query.ts | 4 +- .../src/core/query-executor.ts | 137 ++++++++-- .../src/types/index.ts | 13 + .../src/utils/config-handler.ts | 11 +- .../test/unit/query-executor.test.ts | 42 ++- 13 files changed, 676 insertions(+), 26 deletions(-) create mode 100644 packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts create mode 100644 packages/contentstack-asset-management/src/query-export/index.ts create mode 100644 packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts diff --git a/.talismanrc b/.talismanrc index ed36e71da..3fcbc91dd 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,8 @@ fileignoreconfig: - filename: pnpm-lock.yaml checksum: 2f6edbc19377e3a857884f00e31c498b660cc4f64b46892aee8eecf2f1ca9978 + - filename: packages/contentstack-query-export/src/core/query-executor.ts + checksum: a8d3688a519eb6a941bcdb22f41347df87539e7ab2413665b3193614dc40622d + - filename: packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts + checksum: 19c19d237e1dbe339d024c92049e24493d30b133422d2a10173fe3718413370c version: '1.0' diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts index c66c638d0..c062f899b 100644 --- a/packages/contentstack-asset-management/src/index.ts +++ b/packages/contentstack-asset-management/src/index.ts @@ -3,3 +3,4 @@ export * from './types'; export * from './utils'; export * from './export'; export * from './import'; +export * from './query-export'; diff --git a/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts new file mode 100644 index 000000000..77b110553 --- /dev/null +++ b/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts @@ -0,0 +1,242 @@ +import { resolve as pResolve } from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { Readable } from 'node:stream'; +import { log, handleAndLogError, configHandler } from '@contentstack/cli-utilities'; + +import type { AmAssetQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; +import type { ExportContext } from '../types/export-types'; +import ExportAssetTypes from '../export/asset-types'; +import ExportFields from '../export/fields'; +import { CSAssetsExportAdapter } from '../export/base'; +import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; +import { runInBatches } from '../utils/concurrent-batch'; + +const DEFAULT_ASSET_BATCH_SIZE = 100; +const SEARCH_PAGE_LIMIT = 50; + +/** + * Query-based AM 2.0 asset exporter. + * Exports only referenced asset UIDs from entries into the `spaces/` directory layout. + */ +export class AmAssetQueryExporter { + private readonly options: AmAssetQueryExportOptions; + + constructor(options: AmAssetQueryExportOptions) { + this.options = options; + } + + async export(assetUIDs: string[]): Promise { + const { linkedWorkspaces, exportDir, context } = this.options; + + if (!assetUIDs.length) { + log.info('No asset UIDs to export for AM 2.0 query export', context); + return; + } + + if (!linkedWorkspaces.length) { + log.warn('No linked workspaces configured for AM 2.0 asset query export', context); + return; + } + + log.info( + `Starting AM 2.0 query asset export (${assetUIDs.length} UID(s), ${linkedWorkspaces.length} space(s))`, + context, + ); + + const spacesRootPath = pResolve(exportDir, 'spaces'); + await mkdir(spacesRootPath, { recursive: true }); + + const apiConfig: CSAssetsAPIConfig = { + baseURL: this.options.csAssetsUrl, + headers: { organization_uid: this.options.org_uid }, + context, + }; + + const exportContext: ExportContext = { + spacesRootPath, + context, + securedAssets: this.options.securedAssets, + chunkFileSizeMb: this.options.chunkFileSizeMb, + apiConcurrency: this.options.apiConcurrency, + downloadAssetsConcurrency: this.options.downloadAssetsConcurrency, + }; + + const batchSize = this.options.assetBatchSize ?? DEFAULT_ASSET_BATCH_SIZE; + + try { + await this.bootstrapSharedModules(apiConfig, exportContext, linkedWorkspaces[0].space_uid); + + for (const workspace of linkedWorkspaces) { + try { + await this.exportWorkspaceAssets(apiConfig, exportContext, workspace, assetUIDs, batchSize); + } catch (err) { + handleAndLogError( + err, + { ...(context as Record), spaceUid: workspace.space_uid }, + `Failed AM 2.0 query export for space ${workspace.space_uid}`, + ); + } + } + + log.success('AM 2.0 query asset export completed', context); + } catch (err) { + handleAndLogError(err, context as Record, 'AM 2.0 query asset export failed'); + throw err; + } + } + + private async bootstrapSharedModules( + apiConfig: CSAssetsAPIConfig, + exportContext: ExportContext, + firstSpaceUid: string, + ): Promise { + const sharedFieldsDir = pResolve(exportContext.spacesRootPath, 'fields'); + const sharedAssetTypesDir = pResolve(exportContext.spacesRootPath, 'asset_types'); + await mkdir(sharedFieldsDir, { recursive: true }); + await mkdir(sharedAssetTypesDir, { recursive: true }); + + const exportAssetTypes = new ExportAssetTypes(apiConfig, exportContext); + const exportFields = new ExportFields(apiConfig, exportContext); + await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]); + } + + private async exportWorkspaceAssets( + apiConfig: CSAssetsAPIConfig, + exportContext: ExportContext, + workspace: LinkedWorkspace, + assetUIDs: string[], + batchSize: number, + ): Promise { + const { branchName, context } = this.options; + const workspaceExporter = new QueryExportWorkspaceAdapter(apiConfig, exportContext); + await workspaceExporter.start(workspace, assetUIDs, branchName || 'main', batchSize); + log.debug(`AM 2.0 query export finished for space ${workspace.space_uid}`, context); + } +} + +/** + * Per-space export: search by UID, write metadata/files, download binaries. + */ +class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { + async start( + workspace: LinkedWorkspace, + assetUIDs: string[], + branchName: string, + uidBatchSize: number, + ): Promise { + await this.init(); + + const spaceDir = pResolve(this.exportContext.spacesRootPath, workspace.space_uid); + await mkdir(spaceDir, { recursive: true }); + + const spaceResponse = await this.getSpace(workspace.space_uid); + const space = spaceResponse.space; + const metadata = { + ...space, + workspace_uid: workspace.uid, + is_default: workspace.is_default, + branch: branchName, + }; + await writeFile(pResolve(spaceDir, 'metadata.json'), JSON.stringify(metadata, null, 2)); + + const assetsDir = pResolve(spaceDir, 'assets'); + await mkdir(assetsDir, { recursive: true }); + + const spaceRef = { space_uid: workspace.space_uid, workspace: workspace.uid }; + const assetItems = await this.searchAllAssets(assetUIDs, spaceRef, uidBatchSize); + + const folders = assetItems.filter((item) => (item as { is_dir?: boolean }).is_dir === true); + const files = assetItems.filter((item) => (item as { is_dir?: boolean }).is_dir !== true); + + await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); + + await this.writeItemsToChunkedJson( + assetsDir, + 'assets.json', + 'assets', + ['uid', 'url', 'filename', 'file_name', 'parent_uid'], + files, + ); + + await this.downloadAssets(files, assetsDir, workspace.space_uid); + } + + private async searchAllAssets( + assetUIDs: string[], + spaceRef: { space_uid: string; workspace: string }, + uidBatchSize: number, + ): Promise>> { + const seen = new Set(); + const results: Array> = []; + + for (let i = 0; i < assetUIDs.length; i += uidBatchSize) { + const uidBatch = assetUIDs.slice(i, i + uidBatchSize); + let skip = 0; + let pageItems: unknown[]; + + do { + const response = await this.searchAssets({ + assetUIDs: uidBatch, + spaces: [spaceRef], + skip, + limit: SEARCH_PAGE_LIMIT, + }); + pageItems = getAssetItems(response); + if (pageItems.length === 0 && Array.isArray((response as { assets?: unknown[] }).assets)) { + pageItems = (response as { assets: unknown[] }).assets; + } + + for (const item of pageItems) { + const record = item as Record; + const key = String(record.uid ?? record.asset_id ?? record._uid ?? ''); + if (key && !seen.has(key)) { + seen.add(key); + results.push(record); + } + } + + skip += pageItems.length; + } while (pageItems.length === SEARCH_PAGE_LIMIT); + } + + return results; + } + + private async downloadAssets( + items: Array>, + assetsDir: string, + spaceUid: string, + ): Promise { + const downloadable = items.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))); + if (downloadable.length === 0) { + log.debug(`No downloadable assets for space ${spaceUid}`, this.exportContext.context); + return; + } + + const filesDir = pResolve(assetsDir, 'files'); + await mkdir(filesDir, { recursive: true }); + + const securedAssets = this.exportContext.securedAssets ?? false; + const authtoken = securedAssets ? configHandler.get('authtoken') : null; + + await runInBatches(downloadable, this.downloadAssetsBatchConcurrency, async (asset) => { + const uid = String(asset.uid ?? asset._uid); + const url = String(asset.url); + const filename = String(asset.filename ?? asset.file_name ?? 'asset'); + try { + const separator = url.includes('?') ? '&' : '?'; + const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; + const response = await fetch(downloadUrl); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const body = response.body; + if (!body) throw new Error('No response body'); + const nodeStream = Readable.fromWeb(body as Parameters[0]); + const assetFolderPath = pResolve(filesDir, uid); + await mkdir(assetFolderPath, { recursive: true }); + await writeStreamToFile(nodeStream, pResolve(assetFolderPath, filename)); + } catch (e) { + log.debug(`Failed to download asset ${uid} in space ${spaceUid}: ${e}`, this.exportContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/query-export/index.ts b/packages/contentstack-asset-management/src/query-export/index.ts new file mode 100644 index 000000000..181aca119 --- /dev/null +++ b/packages/contentstack-asset-management/src/query-export/index.ts @@ -0,0 +1 @@ +export { AmAssetQueryExporter } from './am-asset-query-exporter'; diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 6c8c379e5..79f313686 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -119,6 +119,28 @@ export type CSAssetsAPIConfig = { * Adapter interface for Contentstack Assets API calls. * Used by export and (future) import. */ +/** Space + workspace pair for AM search API. */ +export type SearchSpaceRef = { + space_uid: string; + workspace: string; +}; + +/** Parameters for POST /api/search (asset query export). */ +export type SearchAssetsParams = { + assetUIDs: string[]; + spaces: SearchSpaceRef[]; + skip?: number; + limit?: number; +}; + +/** Response shape from POST /api/search for assets. */ +export type SearchAssetsResponse = { + count?: number; + assets?: unknown[]; + items?: unknown[]; + folders?: unknown[]; +}; + export interface ICSAssetsAdapter { init(): Promise; listSpaces(): Promise; @@ -127,8 +149,26 @@ export interface ICSAssetsAdapter { getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; + searchAssets(params: SearchAssetsParams): Promise; } +/** Options for query-based AM asset export (referenced assets from entries). */ +export type AmAssetQueryExportOptions = { + linkedWorkspaces: LinkedWorkspace[]; + exportDir: string; + branchName: string; + csAssetsUrl: string; + org_uid: string; + apiKey?: string; + context?: Record; + securedAssets?: boolean; + chunkFileSizeMb?: number; + apiConcurrency?: number; + downloadAssetsConcurrency?: number; + /** Max UIDs per search request ($in batch). */ + assetBatchSize?: number; +}; + /** * Options for exporting space structure (used by export app after fetching linked workspaces). */ diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index dcdf26a11..2ee4c2230 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -12,11 +12,41 @@ import type { CreateSpacePayload, FieldsResponse, ICSAssetsAdapter, + SearchAssetsParams, + SearchAssetsResponse, Space, SpaceResponse, SpacesListResponse, } from '../types/cs-assets-api'; +/** Default fields requested from POST /api/search for asset export. */ +export const DEFAULT_SEARCH_ASSET_FIELDS = [ + 'asset_id', + 'uid', + 'title', + 'file_name', + 'description', + 'parent_uid', + 'is_dir', + 'dimensions', + 'file_size', + 'content_type', + 'asset_type', + 'url', + 'tags', + 'created_at', + 'updated_at', + 'created_by', + 'updated_by', + 'path', + 'locale', + 'space_uid', + 'version', + 'publish_details', + 'ACL', + '_asset_scan_status', +] as const; + export class CSAssetsAdapter implements ICSAssetsAdapter { private readonly config: CSAssetsAPIConfig; private readonly apiClient: HttpClient; @@ -223,6 +253,30 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } + /** + * POST /api/search — query assets by UID within linked spaces (AM 2.0 query export). + */ + async searchAssets(params: SearchAssetsParams): Promise { + await this.init(); + const { assetUIDs, spaces, skip = 0, limit = 50 } = params; + if (!assetUIDs.length) { + return { count: 0, assets: [] }; + } + const body = { + query: { uid: { $in: assetUIDs } }, + skip, + limit, + object_type: 'asset', + fields: [...DEFAULT_SEARCH_ASSET_FIELDS], + spaces, + }; + log.debug( + `Searching assets (skip=${skip}, limit=${limit}, uids=${assetUIDs.length}, spaces=${spaces.length})`, + this.config.context, + ); + return this.postJson('/api/search', body); + } + // --------------------------------------------------------------------------- // POST helpers // --------------------------------------------------------------------------- diff --git a/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts new file mode 100644 index 000000000..af3d6cae5 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts @@ -0,0 +1,152 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as fs from 'node:fs/promises'; +import { resolve as pResolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { HttpClient, authenticationHandler } from '@contentstack/cli-utilities'; + +import { AmAssetQueryExporter } from '../../../src/query-export/am-asset-query-exporter'; +import ExportAssetTypes from '../../../src/export/asset-types'; +import ExportFields from '../../../src/export/fields'; +import { CSAssetsExportAdapter } from '../../../src/export/base'; +import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; +import * as concurrentBatch from '../../../src/utils/concurrent-batch'; + +import type { AmAssetQueryExportOptions } from '../../../src/types/cs-assets-api'; + +describe('AmAssetQueryExporter', () => { + let exportDir: string; + let searchAssetsStub: sinon.SinonStub; + const baseOptions: AmAssetQueryExportOptions = { + linkedWorkspaces: [{ uid: 'main', space_uid: 'space-1', is_default: true }], + exportDir: '', + branchName: 'main', + csAssetsUrl: 'https://am.example.com', + org_uid: 'org-1', + context: { command: 'export-query' }, + assetBatchSize: 2, + }; + + beforeEach(async () => { + exportDir = await fs.mkdtemp(pResolve(tmpdir(), 'am-query-export-')); + baseOptions.exportDir = exportDir; + + sinon.stub(ExportFields.prototype, 'start').resolves(); + sinon.stub(ExportAssetTypes.prototype, 'start').resolves(); + sinon.stub(CSAssetsExportAdapter.prototype, 'init').resolves(); + sinon.stub(CSAssetsExportAdapter.prototype, 'getSpace').resolves({ + space: { uid: 'space-1', title: 'Test Space' }, + }); + searchAssetsStub = sinon.stub(CSAssetsExportAdapter.prototype, 'searchAssets').resolves({ + assets: [ + { uid: 'asset-1', url: 'https://cdn.example.com/a1.png', file_name: 'a1.png', is_dir: false }, + { uid: 'asset-2', url: 'https://cdn.example.com/a2.png', file_name: 'a2.png', is_dir: false }, + ], + }); + sinon.stub(CSAssetsExportAdapter.prototype as any, 'writeItemsToChunkedJson').resolves(); + sinon.stub(concurrentBatch, 'runInBatches').callsFake(async (items, _concurrency, handler) => { + for (let i = 0; i < items.length; i++) { + await handler(items[i], i); + } + }); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should return early when no asset UIDs are provided', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export([]); + + expect((ExportFields.prototype.start as sinon.SinonStub).called).to.be.false; + }); + + it('should bootstrap shared fields and asset types', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export(['asset-1']); + + expect((ExportFields.prototype.start as sinon.SinonStub).calledOnceWith('space-1')).to.be.true; + expect((ExportAssetTypes.prototype.start as sinon.SinonStub).calledOnceWith('space-1')).to.be.true; + }); + + it('should call searchAssets with batched UIDs and space reference', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export(['asset-1', 'asset-2', 'asset-3']); + + expect(searchAssetsStub.called).to.be.true; + const firstCall = searchAssetsStub.getCall(0).args[0]; + expect(firstCall.spaces).to.deep.equal([{ space_uid: 'space-1', workspace: 'main' }]); + expect(firstCall.assetUIDs).to.deep.equal(['asset-1', 'asset-2']); + }); + + it('should write space metadata and asset files under spaces/', async () => { + const exporter = new AmAssetQueryExporter(baseOptions); + await exporter.export(['asset-1']); + + const metadataPath = pResolve(exportDir, 'spaces', 'space-1', 'metadata.json'); + const metadata = JSON.parse(await fs.readFile(metadataPath, 'utf-8')); + expect(metadata.uid).to.equal('space-1'); + expect(metadata.workspace_uid).to.equal('main'); + + const foldersPath = pResolve(exportDir, 'spaces', 'space-1', 'assets', 'folders.json'); + const folders = JSON.parse(await fs.readFile(foldersPath, 'utf-8')); + expect(folders).to.be.an('array').that.is.empty; + }); +}); + +describe('CSAssetsAdapter.searchAssets', () => { + const baseConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(HttpClient.prototype, 'headers').returnsThis(); + sinon.stub(HttpClient.prototype, 'baseUrl').returnsThis(); + sinon.stub(authenticationHandler, 'getAuthDetails').resolves(); + sinon.stub(authenticationHandler, 'isOauthEnabled').get(() => false); + sinon.stub(authenticationHandler, 'accessToken').get(() => 'test-token'); + + fetchStub = sinon.stub(global, 'fetch').resolves({ + ok: true, + json: async () => ({ count: 1, assets: [{ uid: 'a1' }] }), + } as Response); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should POST to /api/search with uid $in query', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.searchAssets({ + assetUIDs: ['uid-1', 'uid-2'], + spaces: [{ space_uid: 'space-1', workspace: 'main' }], + skip: 0, + limit: 50, + }); + + expect(fetchStub.calledOnce).to.be.true; + const [url, init] = fetchStub.firstCall.args; + expect(url).to.equal('https://am.example.com/api/search'); + expect(init.method).to.equal('POST'); + const body = JSON.parse(init.body); + expect(body.query).to.deep.equal({ uid: { $in: ['uid-1', 'uid-2'] } }); + expect(body.object_type).to.equal('asset'); + expect(body.spaces).to.deep.equal([{ space_uid: 'space-1', workspace: 'main' }]); + }); + + it('should return empty result when assetUIDs is empty', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.searchAssets({ + assetUIDs: [], + spaces: [{ space_uid: 'space-1', workspace: 'main' }], + }); + + expect(fetchStub.called).to.be.false; + expect(result).to.deep.equal({ count: 0, assets: [] }); + }); +}); diff --git a/packages/contentstack-query-export/package.json b/packages/contentstack-query-export/package.json index 1bb79f2a1..e2a32317a 100644 --- a/packages/contentstack-query-export/package.json +++ b/packages/contentstack-query-export/package.json @@ -5,6 +5,7 @@ "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { + "@contentstack/cli-asset-management": "~1.0.0-beta.1", "@contentstack/cli-cm-export": "~2.0.0-beta.17", "@contentstack/cli-command": "~2.0.0-beta.7", "@contentstack/cli-utilities": "~2.0.0-beta.8", diff --git a/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts b/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts index e70e6c2de..240d363ff 100644 --- a/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts +++ b/packages/contentstack-query-export/src/commands/cm/stacks/export-query.ts @@ -12,7 +12,7 @@ import { } from '@contentstack/cli-utilities'; import { QueryExporter } from '../../../core/query-executor'; import { QueryExportConfig } from '../../../types'; -import { setupQueryExportConfig, setupBranches, createLogContext } from '../../../utils'; +import { setupQueryExportConfig, setupBranches, createLogContext, applyRegionToQueryExportConfig } from '../../../utils'; export default class ExportQueryCommand extends Command { static description = 'Export content from a stack using query-based filtering'; @@ -80,7 +80,7 @@ export default class ExportQueryCommand extends Command { // Setup export configuration const exportQueryConfig = await setupQueryExportConfig(flags); exportQueryConfig.host = this.cmaHost; - exportQueryConfig.region = this.region; + applyRegionToQueryExportConfig(exportQueryConfig, this.region); if (this.developerHubUrl) { exportQueryConfig.developerHubBaseUrl = this.developerHubUrl; diff --git a/packages/contentstack-query-export/src/core/query-executor.ts b/packages/contentstack-query-export/src/core/query-executor.ts index dd48183df..0d9ca5f57 100644 --- a/packages/contentstack-query-export/src/core/query-executor.ts +++ b/packages/contentstack-query-export/src/core/query-executor.ts @@ -4,7 +4,9 @@ import { log, handleAndLogError, readContentTypeSchemas, + managementSDKClient, } from '@contentstack/cli-utilities'; +import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; import * as fs from 'fs'; import * as path from 'path'; import { QueryExportConfig, Modules } from '../types'; @@ -44,6 +46,9 @@ export class QueryExporter { // Step 2: Always export general modules await this.exportGeneralModules(); + // Step 3: Resolve AM 2.0 linked workspaces from branch settings + await this.fetchLinkedWorkspaces(); + // Step 4: Export queried modules await this.exportQueriedModule(parsedQuery); @@ -59,6 +64,56 @@ export class QueryExporter { log.success('Query-based export completed successfully!', this.exportQueryConfig.context); } + /** + * Fetch linked workspaces (am_v2) from branch settings for AM 2.0 asset routing. + */ + private async fetchLinkedWorkspaces(): Promise { + const branchName = this.exportQueryConfig.branchName || 'main'; + try { + const branch = await this.stackAPIClient + .branch(branchName) + .fetch({ include_settings: true } as Record); + const linked = (branch as { settings?: { am_v2?: { linked_workspaces?: QueryExportConfig['linkedWorkspaces'] } } }) + ?.settings?.am_v2?.linked_workspaces; + this.exportQueryConfig.linkedWorkspaces = Array.isArray(linked) ? linked : []; + log.debug( + `Linked workspaces for AM 2.0: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, + this.exportQueryConfig.context, + ); + } catch (error) { + log.warn( + `Could not fetch linked workspaces for branch ${branchName}, using legacy asset export`, + this.exportQueryConfig.context, + ); + this.exportQueryConfig.linkedWorkspaces = []; + } + } + + private isAM2AssetExport(): boolean { + return ( + (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && + Boolean(this.exportQueryConfig.csAssetsUrl) + ); + } + + /** + * Resolve organization UID for AM 2.0 API calls. + */ + private async resolveOrgUid(): Promise { + if (this.exportQueryConfig.org_uid) { + return this.exportQueryConfig.org_uid; + } + try { + const tempAPIClient = await managementSDKClient({ host: this.exportQueryConfig.host }); + const stackData = await tempAPIClient.stack({ api_key: this.exportQueryConfig.stackApiKey }).fetch(); + this.exportQueryConfig.org_uid = (stackData as { org_uid?: string })?.org_uid ?? ''; + return this.exportQueryConfig.org_uid; + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Failed to resolve organization UID'); + return ''; + } + } + // export general modules private async exportGeneralModules(): Promise { log.info('Exporting general modules...', this.exportQueryConfig.context); @@ -266,28 +321,75 @@ export class QueryExporter { log.info('Starting export of referenced assets...', this.exportQueryConfig.context); try { - const assetsDir = path.join( - sanitizePath(this.exportQueryConfig.exportDir), - sanitizePath(this.exportQueryConfig.branchName || ''), - 'assets', - ); - - const metadataFilePath = path.join(assetsDir, 'metadata.json'); - const assetFilePath = path.join(assetsDir, 'assets.json'); - - // Define temp file paths - const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json'); - const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); - const assetHandler = new AssetReferenceHandler(this.exportQueryConfig); // Extract referenced asset UIDs from all entries log.debug('Extracting referenced assets from entries', this.exportQueryConfig.context); const assetUIDs = assetHandler.extractReferencedAssets(); - if (assetUIDs.length > 0) { - log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); + if (assetUIDs.length === 0) { + log.info('No referenced assets found in entries', this.exportQueryConfig.context); + return; + } + + log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); + + if (this.isAM2AssetExport()) { + await this.exportReferencedAssetsAM2(assetUIDs); + return; + } + + await this.exportReferencedAssetsLegacy(assetUIDs); + } catch (error) { + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); + throw error; + } + } + + /** + * AM 2.0: export referenced assets into spaces/ via Contentstack Assets search API. + */ + private async exportReferencedAssetsAM2(assetUIDs: string[]): Promise { + log.info('Using AM 2.0 asset export (spaces/)', this.exportQueryConfig.context); + const org_uid = await this.resolveOrgUid(); + if (!org_uid) { + throw new Error('Organization UID is required for AM 2.0 asset export'); + } + const amExporter = new AmAssetQueryExporter({ + linkedWorkspaces: this.exportQueryConfig.linkedWorkspaces ?? [], + exportDir: this.exportQueryConfig.exportDir, + branchName: this.exportQueryConfig.branchName || 'main', + csAssetsUrl: this.exportQueryConfig.csAssetsUrl!, + org_uid, + apiKey: this.exportQueryConfig.stackApiKey, + context: this.exportQueryConfig.context as unknown as Record, + securedAssets: this.exportQueryConfig.securedAssets, + assetBatchSize: this.exportQueryConfig.assetBatchSize, + }); + + await amExporter.export(assetUIDs); + log.success('Referenced assets exported successfully (AM 2.0)', this.exportQueryConfig.context); + } + + /** + * AM 1.0: export referenced assets into legacy assets/ via CMA export module. + */ + private async exportReferencedAssetsLegacy(assetUIDs: string[]): Promise { + const assetsDir = path.join( + sanitizePath(this.exportQueryConfig.exportDir), + sanitizePath(this.exportQueryConfig.branchName || ''), + 'assets', + ); + + const metadataFilePath = path.join(assetsDir, 'metadata.json'); + const assetFilePath = path.join(assetsDir, 'assets.json'); + + // Define temp file paths + const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json'); + const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); + + try { fs.mkdirSync(assetsDir, { recursive: true }); // Define batch size - can be configurable through exportQueryConfig @@ -383,11 +485,8 @@ export class QueryExporter { log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); log.success('Referenced assets exported successfully', this.exportQueryConfig.context); - } else { - log.info('No referenced assets found in entries', this.exportQueryConfig.context); - } } catch (error) { - handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting legacy referenced assets'); throw error; } } diff --git a/packages/contentstack-query-export/src/types/index.ts b/packages/contentstack-query-export/src/types/index.ts index 1a036ea24..6c96ce564 100644 --- a/packages/contentstack-query-export/src/types/index.ts +++ b/packages/contentstack-query-export/src/types/index.ts @@ -31,8 +31,15 @@ export interface Region { cma: string; cda: string; uiHost: string; + csAssetsUrl?: string; } +export type LinkedWorkspace = { + uid: string; + space_uid: string; + is_default: boolean; +}; + export type Modules = | 'stack' | 'locales' @@ -203,6 +210,12 @@ export interface QueryExportConfig extends DefaultConfig { batchDelayMs?: number; assetBatchSize?: number; assetBatchDelayMs?: number; + /** AM 2.0 linked workspaces from branch settings (am_v2.linked_workspaces). */ + linkedWorkspaces?: LinkedWorkspace[]; + /** Contentstack Assets API base URL for AM 2.0 export. */ + csAssetsUrl?: string; + /** Organization UID for AM 2.0 API headers. */ + org_uid?: string; context?: LogContext; // Log context for centralized logging } diff --git a/packages/contentstack-query-export/src/utils/config-handler.ts b/packages/contentstack-query-export/src/utils/config-handler.ts index 94eccfc65..1a3ba02a4 100644 --- a/packages/contentstack-query-export/src/utils/config-handler.ts +++ b/packages/contentstack-query-export/src/utils/config-handler.ts @@ -1,9 +1,18 @@ import * as path from 'path'; -import { QueryExportConfig } from '../types'; +import { QueryExportConfig, Region } from '../types'; import { sanitizePath, pathValidator, configHandler, isAuthenticated } from '@contentstack/cli-utilities'; import config from '../config'; import { askAPIKey } from './common-helper'; +/** + * Apply region-derived AM 2.0 settings after the command resolves region. + */ +export function applyRegionToQueryExportConfig(exportQueryConfig: QueryExportConfig, region?: Region): void { + if (!region) return; + exportQueryConfig.region = region; + exportQueryConfig.csAssetsUrl = region.csAssetsUrl; +} + export async function setupQueryExportConfig(flags: any): Promise { const exportDir = sanitizePath(flags['data-dir'] || pathValidator('export')); diff --git a/packages/contentstack-query-export/test/unit/query-executor.test.ts b/packages/contentstack-query-export/test/unit/query-executor.test.ts index a3cfd4bc6..a7d3b3c7c 100644 --- a/packages/contentstack-query-export/test/unit/query-executor.test.ts +++ b/packages/contentstack-query-export/test/unit/query-executor.test.ts @@ -3,6 +3,7 @@ import * as sinon from 'sinon'; import { QueryExporter } from '../../src/core/query-executor'; import { QueryParser } from '../../src/utils/query-parser'; import { ModuleExporter } from '../../src/core/module-exporter'; +import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; import * as logger from '../../src/utils/logger'; import { ReferencedContentTypesHandler, @@ -23,7 +24,11 @@ describe('QueryExporter', () => { // Mock management client mockManagementClient = { - stack: sandbox.stub().returns({}), + stack: sandbox.stub().returns({ + branch: sandbox.stub().returns({ + fetch: sandbox.stub().resolves({ settings: { am_v2: { linked_workspaces: [] } } }), + }), + }), }; // Mock export configuration @@ -74,6 +79,7 @@ describe('QueryExporter', () => { describe('execute', () => { let queryParserStub: sinon.SinonStub; let exportGeneralModulesStub: sinon.SinonStub; + let fetchLinkedWorkspacesStub: sinon.SinonStub; let exportQueriedModuleStub: sinon.SinonStub; let expandSchemaClosureStub: sinon.SinonStub; let exportContentModulesStub: sinon.SinonStub; @@ -83,6 +89,7 @@ describe('QueryExporter', () => { modules: { entries: { content_type_uid: 'test_page' } }, }); exportGeneralModulesStub = sandbox.stub(queryExporter as any, 'exportGeneralModules').resolves(); + fetchLinkedWorkspacesStub = sandbox.stub(queryExporter as any, 'fetchLinkedWorkspaces').resolves(); exportQueriedModuleStub = sandbox.stub(queryExporter as any, 'exportQueriedModule').resolves(); expandSchemaClosureStub = sandbox.stub(queryExporter as any, 'expandSchemaClosure').resolves(); exportContentModulesStub = sandbox.stub(queryExporter as any, 'exportContentModules').resolves(); @@ -104,6 +111,7 @@ describe('QueryExporter', () => { sinon.assert.callOrder( queryParserStub, exportGeneralModulesStub, + fetchLinkedWorkspacesStub, exportQueriedModuleStub, expandSchemaClosureStub, exportContentModulesStub, @@ -569,9 +577,11 @@ describe('QueryExporter', () => { describe('exportReferencedAssets', () => { let moduleExporterStub: sinon.SinonStub; let assetHandlerStub: any; + let amExporterStub: sinon.SinonStub; beforeEach(() => { moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); + amExporterStub = sandbox.stub(AmAssetQueryExporter.prototype, 'export').resolves(); // Mock AssetReferenceHandler assetHandlerStub = { @@ -582,15 +592,39 @@ describe('QueryExporter', () => { .callsFake(assetHandlerStub.extractReferencedAssets); }); - it('should export referenced assets when found', async () => { + it('should export referenced assets when found (AM 1.0 legacy path)', async () => { + mockConfig.linkedWorkspaces = []; + mockConfig.csAssetsUrl = undefined; + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + const legacyModuleExporterStub = sandbox + .stub((queryExporter as any).moduleExporter, 'exportModule') + .resolves(); + await (queryExporter as any).exportReferencedAssets(); - expect(moduleExporterStub.calledOnce).to.be.true; - const exportCall = moduleExporterStub.getCall(0); + expect(legacyModuleExporterStub.calledOnce).to.be.true; + expect(amExporterStub.called).to.be.false; + const exportCall = legacyModuleExporterStub.getCall(0); expect(exportCall.args[0]).to.equal('assets'); expect(exportCall.args[1].query.modules.assets.uid.$in).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); }); + it('should use AmAssetQueryExporter when AM 2.0 linked workspaces and csAssetsUrl are set', async () => { + mockConfig.linkedWorkspaces = [{ uid: 'main', space_uid: 'space-1', is_default: true }]; + mockConfig.csAssetsUrl = 'https://am.example.com'; + mockConfig.org_uid = 'org-1'; + queryExporter = new QueryExporter(mockManagementClient, mockConfig); + const am2ModuleExporterStub = sandbox + .stub((queryExporter as any).moduleExporter, 'exportModule') + .resolves(); + + await (queryExporter as any).exportReferencedAssets(); + + expect(amExporterStub.calledOnce).to.be.true; + expect(amExporterStub.firstCall.args[0]).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); + expect(am2ModuleExporterStub.called).to.be.false; + }); + it('should skip export when no assets found', async () => { assetHandlerStub.extractReferencedAssets.returns([]); From ab3991a2e4c0c91654514fb43d6cd2de1ba86a30 Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 18 May 2026 12:32:35 +0530 Subject: [PATCH 2/5] updated unit test pipeline --- .github/workflows/unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 62c048f9b..7e7119677 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -25,7 +25,7 @@ jobs: - name: Prune pnpm store run: pnpm store prune - name: Install Dependencies - run: pnpm install --frozen-lockfile + run: pnpm install --no-frozen-lockfile - name: Build all plugins run: | From b007415f17e7aa0c37bc9ac7693d770495c4768d Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 18 May 2026 13:15:08 +0530 Subject: [PATCH 3/5] updated naming --- ...xporter.ts => cs-assets-query-exporter.ts} | 24 +++++------ .../src/query-export/index.ts | 2 +- .../src/types/cs-assets-api.ts | 6 +-- .../src/utils/cs-assets-api-adapter.ts | 2 +- ...st.ts => cs-assets-query-exporter.test.ts} | 18 ++++----- .../src/core/query-executor.ts | 40 +++++++++---------- .../src/types/index.ts | 6 +-- .../src/utils/config-handler.ts | 2 +- .../test/unit/query-executor.test.ts | 20 +++++----- 9 files changed, 60 insertions(+), 60 deletions(-) rename packages/contentstack-asset-management/src/query-export/{am-asset-query-exporter.ts => cs-assets-query-exporter.ts} (89%) rename packages/contentstack-asset-management/test/unit/query-export/{am-asset-query-exporter.test.ts => cs-assets-query-exporter.test.ts} (90%) diff --git a/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts similarity index 89% rename from packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts rename to packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts index 77b110553..f66487bb7 100644 --- a/packages/contentstack-asset-management/src/query-export/am-asset-query-exporter.ts +++ b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts @@ -3,7 +3,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { Readable } from 'node:stream'; import { log, handleAndLogError, configHandler } from '@contentstack/cli-utilities'; -import type { AmAssetQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; +import type { CsAssetsQueryExportOptions, CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import ExportAssetTypes from '../export/asset-types'; import ExportFields from '../export/fields'; @@ -15,13 +15,13 @@ const DEFAULT_ASSET_BATCH_SIZE = 100; const SEARCH_PAGE_LIMIT = 50; /** - * Query-based AM 2.0 asset exporter. + * Query-based Contentstack Assets exporter. * Exports only referenced asset UIDs from entries into the `spaces/` directory layout. */ -export class AmAssetQueryExporter { - private readonly options: AmAssetQueryExportOptions; +export class CsAssetsQueryExporter { + private readonly options: CsAssetsQueryExportOptions; - constructor(options: AmAssetQueryExportOptions) { + constructor(options: CsAssetsQueryExportOptions) { this.options = options; } @@ -29,17 +29,17 @@ export class AmAssetQueryExporter { const { linkedWorkspaces, exportDir, context } = this.options; if (!assetUIDs.length) { - log.info('No asset UIDs to export for AM 2.0 query export', context); + log.info('No asset UIDs to export for Contentstack Assets query export', context); return; } if (!linkedWorkspaces.length) { - log.warn('No linked workspaces configured for AM 2.0 asset query export', context); + log.warn('No linked workspaces configured for Contentstack Assets query export', context); return; } log.info( - `Starting AM 2.0 query asset export (${assetUIDs.length} UID(s), ${linkedWorkspaces.length} space(s))`, + `Starting Contentstack Assets query export (${assetUIDs.length} UID(s), ${linkedWorkspaces.length} space(s))`, context, ); @@ -73,14 +73,14 @@ export class AmAssetQueryExporter { handleAndLogError( err, { ...(context as Record), spaceUid: workspace.space_uid }, - `Failed AM 2.0 query export for space ${workspace.space_uid}`, + `Failed Contentstack Assets query export for space ${workspace.space_uid}`, ); } } - log.success('AM 2.0 query asset export completed', context); + log.success('Contentstack Assets query export completed', context); } catch (err) { - handleAndLogError(err, context as Record, 'AM 2.0 query asset export failed'); + handleAndLogError(err, context as Record, 'Contentstack Assets query export failed'); throw err; } } @@ -110,7 +110,7 @@ export class AmAssetQueryExporter { const { branchName, context } = this.options; const workspaceExporter = new QueryExportWorkspaceAdapter(apiConfig, exportContext); await workspaceExporter.start(workspace, assetUIDs, branchName || 'main', batchSize); - log.debug(`AM 2.0 query export finished for space ${workspace.space_uid}`, context); + log.debug(`Contentstack Assets query export finished for space ${workspace.space_uid}`, context); } } diff --git a/packages/contentstack-asset-management/src/query-export/index.ts b/packages/contentstack-asset-management/src/query-export/index.ts index 181aca119..a46638b88 100644 --- a/packages/contentstack-asset-management/src/query-export/index.ts +++ b/packages/contentstack-asset-management/src/query-export/index.ts @@ -1 +1 @@ -export { AmAssetQueryExporter } from './am-asset-query-exporter'; +export { CsAssetsQueryExporter } from './cs-assets-query-exporter'; diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 79f313686..cab8389d9 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -119,7 +119,7 @@ export type CSAssetsAPIConfig = { * Adapter interface for Contentstack Assets API calls. * Used by export and (future) import. */ -/** Space + workspace pair for AM search API. */ +/** Space + workspace pair for Contentstack Assets search API. */ export type SearchSpaceRef = { space_uid: string; workspace: string; @@ -152,8 +152,8 @@ export interface ICSAssetsAdapter { searchAssets(params: SearchAssetsParams): Promise; } -/** Options for query-based AM asset export (referenced assets from entries). */ -export type AmAssetQueryExportOptions = { +/** Options for query-based Contentstack Assets export (referenced assets from entries). */ +export type CsAssetsQueryExportOptions = { linkedWorkspaces: LinkedWorkspace[]; exportDir: string; branchName: string; diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index 2ee4c2230..aa33703a8 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -254,7 +254,7 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } /** - * POST /api/search — query assets by UID within linked spaces (AM 2.0 query export). + * POST /api/search — query assets by UID within linked spaces (Contentstack Assets query export). */ async searchAssets(params: SearchAssetsParams): Promise { await this.init(); diff --git a/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts similarity index 90% rename from packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts rename to packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts index af3d6cae5..5c8ddd4ba 100644 --- a/packages/contentstack-asset-management/test/unit/query-export/am-asset-query-exporter.test.ts +++ b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts @@ -5,19 +5,19 @@ import { resolve as pResolve } from 'node:path'; import { tmpdir } from 'node:os'; import { HttpClient, authenticationHandler } from '@contentstack/cli-utilities'; -import { AmAssetQueryExporter } from '../../../src/query-export/am-asset-query-exporter'; +import { CsAssetsQueryExporter } from '../../../src/query-export/cs-assets-query-exporter'; import ExportAssetTypes from '../../../src/export/asset-types'; import ExportFields from '../../../src/export/fields'; import { CSAssetsExportAdapter } from '../../../src/export/base'; import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; import * as concurrentBatch from '../../../src/utils/concurrent-batch'; -import type { AmAssetQueryExportOptions } from '../../../src/types/cs-assets-api'; +import type { CsAssetsQueryExportOptions } from '../../../src/types/cs-assets-api'; -describe('AmAssetQueryExporter', () => { +describe('CsAssetsQueryExporter', () => { let exportDir: string; let searchAssetsStub: sinon.SinonStub; - const baseOptions: AmAssetQueryExportOptions = { + const baseOptions: CsAssetsQueryExportOptions = { linkedWorkspaces: [{ uid: 'main', space_uid: 'space-1', is_default: true }], exportDir: '', branchName: 'main', @@ -28,7 +28,7 @@ describe('AmAssetQueryExporter', () => { }; beforeEach(async () => { - exportDir = await fs.mkdtemp(pResolve(tmpdir(), 'am-query-export-')); + exportDir = await fs.mkdtemp(pResolve(tmpdir(), 'cs-assets-query-export-')); baseOptions.exportDir = exportDir; sinon.stub(ExportFields.prototype, 'start').resolves(); @@ -56,14 +56,14 @@ describe('AmAssetQueryExporter', () => { }); it('should return early when no asset UIDs are provided', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export([]); expect((ExportFields.prototype.start as sinon.SinonStub).called).to.be.false; }); it('should bootstrap shared fields and asset types', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export(['asset-1']); expect((ExportFields.prototype.start as sinon.SinonStub).calledOnceWith('space-1')).to.be.true; @@ -71,7 +71,7 @@ describe('AmAssetQueryExporter', () => { }); it('should call searchAssets with batched UIDs and space reference', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export(['asset-1', 'asset-2', 'asset-3']); expect(searchAssetsStub.called).to.be.true; @@ -81,7 +81,7 @@ describe('AmAssetQueryExporter', () => { }); it('should write space metadata and asset files under spaces/', async () => { - const exporter = new AmAssetQueryExporter(baseOptions); + const exporter = new CsAssetsQueryExporter(baseOptions); await exporter.export(['asset-1']); const metadataPath = pResolve(exportDir, 'spaces', 'space-1', 'metadata.json'); diff --git a/packages/contentstack-query-export/src/core/query-executor.ts b/packages/contentstack-query-export/src/core/query-executor.ts index 0d9ca5f57..bc0042149 100644 --- a/packages/contentstack-query-export/src/core/query-executor.ts +++ b/packages/contentstack-query-export/src/core/query-executor.ts @@ -6,7 +6,7 @@ import { readContentTypeSchemas, managementSDKClient, } from '@contentstack/cli-utilities'; -import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; +import { CsAssetsQueryExporter } from '@contentstack/cli-asset-management'; import * as fs from 'fs'; import * as path from 'path'; import { QueryExportConfig, Modules } from '../types'; @@ -46,7 +46,7 @@ export class QueryExporter { // Step 2: Always export general modules await this.exportGeneralModules(); - // Step 3: Resolve AM 2.0 linked workspaces from branch settings + // Step 3: Resolve linked workspaces from branch settings (Contentstack Assets) await this.fetchLinkedWorkspaces(); // Step 4: Export queried modules @@ -65,7 +65,7 @@ export class QueryExporter { } /** - * Fetch linked workspaces (am_v2) from branch settings for AM 2.0 asset routing. + * Fetch linked workspaces from branch settings for Contentstack Assets export routing. */ private async fetchLinkedWorkspaces(): Promise { const branchName = this.exportQueryConfig.branchName || 'main'; @@ -77,19 +77,19 @@ export class QueryExporter { ?.settings?.am_v2?.linked_workspaces; this.exportQueryConfig.linkedWorkspaces = Array.isArray(linked) ? linked : []; log.debug( - `Linked workspaces for AM 2.0: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, + `Linked workspaces for Contentstack Assets: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, this.exportQueryConfig.context, ); } catch (error) { log.warn( - `Could not fetch linked workspaces for branch ${branchName}, using legacy asset export`, + `Could not fetch linked workspaces for branch ${branchName}, using stack assets export`, this.exportQueryConfig.context, ); this.exportQueryConfig.linkedWorkspaces = []; } } - private isAM2AssetExport(): boolean { + private isCsAssetsExport(): boolean { return ( (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && Boolean(this.exportQueryConfig.csAssetsUrl) @@ -97,7 +97,7 @@ export class QueryExporter { } /** - * Resolve organization UID for AM 2.0 API calls. + * Resolve organization UID for Contentstack Assets API calls. */ private async resolveOrgUid(): Promise { if (this.exportQueryConfig.org_uid) { @@ -334,12 +334,12 @@ export class QueryExporter { log.info(`Found ${assetUIDs.length} referenced assets to export`, this.exportQueryConfig.context); - if (this.isAM2AssetExport()) { - await this.exportReferencedAssetsAM2(assetUIDs); + if (this.isCsAssetsExport()) { + await this.exportReferencedCsAssets(assetUIDs); return; } - await this.exportReferencedAssetsLegacy(assetUIDs); + await this.exportReferencedStackAssets(assetUIDs); } catch (error) { handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting referenced assets'); throw error; @@ -347,16 +347,16 @@ export class QueryExporter { } /** - * AM 2.0: export referenced assets into spaces/ via Contentstack Assets search API. + * Export referenced assets into spaces/ via Contentstack Assets search API. */ - private async exportReferencedAssetsAM2(assetUIDs: string[]): Promise { - log.info('Using AM 2.0 asset export (spaces/)', this.exportQueryConfig.context); + private async exportReferencedCsAssets(assetUIDs: string[]): Promise { + log.info('Using Contentstack Assets export (spaces/)', this.exportQueryConfig.context); const org_uid = await this.resolveOrgUid(); if (!org_uid) { - throw new Error('Organization UID is required for AM 2.0 asset export'); + throw new Error('Organization UID is required for Contentstack Assets export'); } - const amExporter = new AmAssetQueryExporter({ + const csAssetsExporter = new CsAssetsQueryExporter({ linkedWorkspaces: this.exportQueryConfig.linkedWorkspaces ?? [], exportDir: this.exportQueryConfig.exportDir, branchName: this.exportQueryConfig.branchName || 'main', @@ -368,14 +368,14 @@ export class QueryExporter { assetBatchSize: this.exportQueryConfig.assetBatchSize, }); - await amExporter.export(assetUIDs); - log.success('Referenced assets exported successfully (AM 2.0)', this.exportQueryConfig.context); + await csAssetsExporter.export(assetUIDs); + log.success('Referenced assets exported successfully (Contentstack Assets)', this.exportQueryConfig.context); } /** - * AM 1.0: export referenced assets into legacy assets/ via CMA export module. + * Export referenced assets into stack assets/ via CMA export module. */ - private async exportReferencedAssetsLegacy(assetUIDs: string[]): Promise { + private async exportReferencedStackAssets(assetUIDs: string[]): Promise { const assetsDir = path.join( sanitizePath(this.exportQueryConfig.exportDir), sanitizePath(this.exportQueryConfig.branchName || ''), @@ -486,7 +486,7 @@ export class QueryExporter { log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); log.success('Referenced assets exported successfully', this.exportQueryConfig.context); } catch (error) { - handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting legacy referenced assets'); + handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting stack referenced assets'); throw error; } } diff --git a/packages/contentstack-query-export/src/types/index.ts b/packages/contentstack-query-export/src/types/index.ts index 6c96ce564..8791069e9 100644 --- a/packages/contentstack-query-export/src/types/index.ts +++ b/packages/contentstack-query-export/src/types/index.ts @@ -210,11 +210,11 @@ export interface QueryExportConfig extends DefaultConfig { batchDelayMs?: number; assetBatchSize?: number; assetBatchDelayMs?: number; - /** AM 2.0 linked workspaces from branch settings (am_v2.linked_workspaces). */ + /** Linked workspaces from branch settings (Contentstack Assets). */ linkedWorkspaces?: LinkedWorkspace[]; - /** Contentstack Assets API base URL for AM 2.0 export. */ + /** Contentstack Assets API base URL. */ csAssetsUrl?: string; - /** Organization UID for AM 2.0 API headers. */ + /** Organization UID for Contentstack Assets API headers. */ org_uid?: string; context?: LogContext; // Log context for centralized logging } diff --git a/packages/contentstack-query-export/src/utils/config-handler.ts b/packages/contentstack-query-export/src/utils/config-handler.ts index 1a3ba02a4..c281c07b0 100644 --- a/packages/contentstack-query-export/src/utils/config-handler.ts +++ b/packages/contentstack-query-export/src/utils/config-handler.ts @@ -5,7 +5,7 @@ import config from '../config'; import { askAPIKey } from './common-helper'; /** - * Apply region-derived AM 2.0 settings after the command resolves region. + * Apply region-derived Contentstack Assets settings after the command resolves region. */ export function applyRegionToQueryExportConfig(exportQueryConfig: QueryExportConfig, region?: Region): void { if (!region) return; diff --git a/packages/contentstack-query-export/test/unit/query-executor.test.ts b/packages/contentstack-query-export/test/unit/query-executor.test.ts index a7d3b3c7c..ecf9670d9 100644 --- a/packages/contentstack-query-export/test/unit/query-executor.test.ts +++ b/packages/contentstack-query-export/test/unit/query-executor.test.ts @@ -3,7 +3,7 @@ import * as sinon from 'sinon'; import { QueryExporter } from '../../src/core/query-executor'; import { QueryParser } from '../../src/utils/query-parser'; import { ModuleExporter } from '../../src/core/module-exporter'; -import { AmAssetQueryExporter } from '@contentstack/cli-asset-management'; +import { CsAssetsQueryExporter } from '@contentstack/cli-asset-management'; import * as logger from '../../src/utils/logger'; import { ReferencedContentTypesHandler, @@ -577,11 +577,11 @@ describe('QueryExporter', () => { describe('exportReferencedAssets', () => { let moduleExporterStub: sinon.SinonStub; let assetHandlerStub: any; - let amExporterStub: sinon.SinonStub; + let csAssetsExporterStub: sinon.SinonStub; beforeEach(() => { moduleExporterStub = sandbox.stub((queryExporter as any).moduleExporter, 'exportModule').resolves(); - amExporterStub = sandbox.stub(AmAssetQueryExporter.prototype, 'export').resolves(); + csAssetsExporterStub = sandbox.stub(CsAssetsQueryExporter.prototype, 'export').resolves(); // Mock AssetReferenceHandler assetHandlerStub = { @@ -592,7 +592,7 @@ describe('QueryExporter', () => { .callsFake(assetHandlerStub.extractReferencedAssets); }); - it('should export referenced assets when found (AM 1.0 legacy path)', async () => { + it('should export referenced assets when found (stack assets path)', async () => { mockConfig.linkedWorkspaces = []; mockConfig.csAssetsUrl = undefined; queryExporter = new QueryExporter(mockManagementClient, mockConfig); @@ -603,26 +603,26 @@ describe('QueryExporter', () => { await (queryExporter as any).exportReferencedAssets(); expect(legacyModuleExporterStub.calledOnce).to.be.true; - expect(amExporterStub.called).to.be.false; + expect(csAssetsExporterStub.called).to.be.false; const exportCall = legacyModuleExporterStub.getCall(0); expect(exportCall.args[0]).to.equal('assets'); expect(exportCall.args[1].query.modules.assets.uid.$in).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); }); - it('should use AmAssetQueryExporter when AM 2.0 linked workspaces and csAssetsUrl are set', async () => { + it('should use CsAssetsQueryExporter when linked workspaces and csAssetsUrl are set', async () => { mockConfig.linkedWorkspaces = [{ uid: 'main', space_uid: 'space-1', is_default: true }]; mockConfig.csAssetsUrl = 'https://am.example.com'; mockConfig.org_uid = 'org-1'; queryExporter = new QueryExporter(mockManagementClient, mockConfig); - const am2ModuleExporterStub = sandbox + const csAssetsModuleExporterStub = sandbox .stub((queryExporter as any).moduleExporter, 'exportModule') .resolves(); await (queryExporter as any).exportReferencedAssets(); - expect(amExporterStub.calledOnce).to.be.true; - expect(amExporterStub.firstCall.args[0]).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); - expect(am2ModuleExporterStub.called).to.be.false; + expect(csAssetsExporterStub.calledOnce).to.be.true; + expect(csAssetsExporterStub.firstCall.args[0]).to.deep.equal(['asset_1', 'asset_2', 'asset_3']); + expect(csAssetsModuleExporterStub.called).to.be.false; }); it('should skip export when no assets found', async () => { From d72c15c4bc6243c80d81067b267dfc6ab611abc1 Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Mon, 1 Jun 2026 12:14:40 +0530 Subject: [PATCH 4/5] fixed am support in export query --- .talismanrc | 2 + .../query-export/cs-assets-query-exporter.ts | 10 +- .../src/types/cs-assets-api.ts | 2 + .../src/utils/cs-assets-api-adapter.ts | 8 +- .../src/utils/export-helpers.ts | 2 +- .../cs-assets-query-exporter.test.ts | 12 +- .../src/export/modules/assets.ts | 1 - .../src/export/modules/publishing-rules.ts | 1 - .../src/core/query-executor.ts | 202 +++++++++--------- 9 files changed, 126 insertions(+), 114 deletions(-) diff --git a/.talismanrc b/.talismanrc index 1a0004955..5793f26e4 100644 --- a/.talismanrc +++ b/.talismanrc @@ -39,4 +39,6 @@ fileignoreconfig: checksum: 9fa87dc7639e821411fba5b4f7871384dc2c4d5c11887a5d3c5a19702e7130be - filename: packages/contentstack-asset-management/test/unit/import-setup/import-setup-asset-mappers.test.ts checksum: a3841d92aed9e31aea8f243c2fa51023cacd619eeac7c35048621247145decb4 + - filename: packages/contentstack-query-export/src/core/query-executor.ts + checksum: 708f8a9bc837ed15342fe73920588978a97cab9002c401dbc6ad7030e0238f48 version: '1.0' diff --git a/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts index f66487bb7..9e50ae94b 100644 --- a/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts +++ b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts @@ -12,7 +12,7 @@ import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; import { runInBatches } from '../utils/concurrent-batch'; const DEFAULT_ASSET_BATCH_SIZE = 100; -const SEARCH_PAGE_LIMIT = 50; +const SEARCH_PAGE_LIMIT = 100; /** * Query-based Contentstack Assets exporter. @@ -182,8 +182,12 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { limit: SEARCH_PAGE_LIMIT, }); pageItems = getAssetItems(response); - if (pageItems.length === 0 && Array.isArray((response as { assets?: unknown[] }).assets)) { - pageItems = (response as { assets: unknown[] }).assets; + + if (pageItems.length === 0 && skip === 0) { + log.warn( + `Search returned 0 assets in space ${spaceRef.space_uid} for UID(s): [${uidBatch.join(', ')}]`, + this.exportContext.context, + ); } for (const item of pageItems) { diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index cab8389d9..38c953580 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -136,8 +136,10 @@ export type SearchAssetsParams = { /** Response shape from POST /api/search for assets. */ export type SearchAssetsResponse = { count?: number; + relation?: string; assets?: unknown[]; items?: unknown[]; + results?: unknown[]; folders?: unknown[]; }; diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index aa33703a8..ff1078e91 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -263,10 +263,16 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return { count: 0, assets: [] }; } const body = { - query: { uid: { $in: assetUIDs } }, + query: { + $and: [{ uid: { $in: assetUIDs } }], + }, skip, limit, + desc: 'updated_at', + search_text: '', + search_field: 'all', object_type: 'asset', + search_terms_operator: 'or', fields: [...DEFAULT_SEARCH_ASSET_FIELDS], spaces, }; diff --git a/packages/contentstack-asset-management/src/utils/export-helpers.ts b/packages/contentstack-asset-management/src/utils/export-helpers.ts index 9bc772c0f..252b33652 100644 --- a/packages/contentstack-asset-management/src/utils/export-helpers.ts +++ b/packages/contentstack-asset-management/src/utils/export-helpers.ts @@ -14,7 +14,7 @@ export function getAssetItems( ): Array<{ uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }> { if (Array.isArray(assetsData)) return assetsData; const data = assetsData as Record; - const items = data?.items ?? data?.assets; + const items = data?.items ?? data?.assets ?? data?.results; return Array.isArray(items) ? items : []; } diff --git a/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts index 5c8ddd4ba..7d5af5090 100644 --- a/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts +++ b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts @@ -38,7 +38,9 @@ describe('CsAssetsQueryExporter', () => { space: { uid: 'space-1', title: 'Test Space' }, }); searchAssetsStub = sinon.stub(CSAssetsExportAdapter.prototype, 'searchAssets').resolves({ - assets: [ + count: 2, + relation: 'eq', + results: [ { uid: 'asset-1', url: 'https://cdn.example.com/a1.png', file_name: 'a1.png', is_dir: false }, { uid: 'asset-2', url: 'https://cdn.example.com/a2.png', file_name: 'a2.png', is_dir: false }, ], @@ -120,7 +122,7 @@ describe('CSAssetsAdapter.searchAssets', () => { sinon.restore(); }); - it('should POST to /api/search with uid $in query', async () => { + it('should POST to /api/search with $and-wrapped uid $in query and required fields', async () => { const adapter = new CSAssetsAdapter(baseConfig); await adapter.searchAssets({ assetUIDs: ['uid-1', 'uid-2'], @@ -134,8 +136,12 @@ describe('CSAssetsAdapter.searchAssets', () => { expect(url).to.equal('https://am.example.com/api/search'); expect(init.method).to.equal('POST'); const body = JSON.parse(init.body); - expect(body.query).to.deep.equal({ uid: { $in: ['uid-1', 'uid-2'] } }); + expect(body.query).to.deep.equal({ $and: [{ uid: { $in: ['uid-1', 'uid-2'] } }] }); expect(body.object_type).to.equal('asset'); + expect(body.desc).to.equal('updated_at'); + expect(body.search_text).to.equal(''); + expect(body.search_field).to.equal('all'); + expect(body.search_terms_operator).to.equal('or'); expect(body.spaces).to.deep.equal([{ space_uid: 'space-1', workspace: 'main' }]); }); diff --git a/packages/contentstack-export/src/export/modules/assets.ts b/packages/contentstack-export/src/export/modules/assets.ts index 6b79da510..a5a529ad9 100644 --- a/packages/contentstack-export/src/export/modules/assets.ts +++ b/packages/contentstack-export/src/export/modules/assets.ts @@ -123,7 +123,6 @@ export default class ExportAssets extends BaseClass { this.assetsRootPath = pResolve( this.exportConfig.exportDir, - this.exportConfig.branchName || '', (this.assetsRootPath = pResolve(getExportBasePath(this.exportConfig), this.assetConfig.dirName)), ); log.debug(`Assets root path resolved to: ${this.assetsRootPath}`, this.exportConfig.context); diff --git a/packages/contentstack-export/src/export/modules/publishing-rules.ts b/packages/contentstack-export/src/export/modules/publishing-rules.ts index b1b045e0a..bd81c3ee2 100644 --- a/packages/contentstack-export/src/export/modules/publishing-rules.ts +++ b/packages/contentstack-export/src/export/modules/publishing-rules.ts @@ -23,7 +23,6 @@ export default class ExportPublishingRules extends BaseClass { async start(): Promise { this.publishingRulesFolderPath = pResolve( this.exportConfig.exportDir, - this.exportConfig.branchName || '', this.publishingRulesConfig.dirName, ); log.debug(`Publishing rules folder path: ${this.publishingRulesFolderPath}`, this.exportConfig.context); diff --git a/packages/contentstack-query-export/src/core/query-executor.ts b/packages/contentstack-query-export/src/core/query-executor.ts index fabc68b8b..607ad89d1 100644 --- a/packages/contentstack-query-export/src/core/query-executor.ts +++ b/packages/contentstack-query-export/src/core/query-executor.ts @@ -73,8 +73,9 @@ export class QueryExporter { const branch = await this.stackAPIClient .branch(branchName) .fetch({ include_settings: true } as Record); - const linked = (branch as { settings?: { am_v2?: { linked_workspaces?: QueryExportConfig['linkedWorkspaces'] } } }) - ?.settings?.am_v2?.linked_workspaces; + const linked = ( + branch as { settings?: { am_v2?: { linked_workspaces?: QueryExportConfig['linkedWorkspaces'] } } } + )?.settings?.am_v2?.linked_workspaces; this.exportQueryConfig.linkedWorkspaces = Array.isArray(linked) ? linked : []; log.debug( `Linked workspaces for Contentstack Assets: ${this.exportQueryConfig.linkedWorkspaces?.length ?? 0}`, @@ -90,10 +91,7 @@ export class QueryExporter { } private isCsAssetsExport(): boolean { - return ( - (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && - Boolean(this.exportQueryConfig.csAssetsUrl) - ); + return (this.exportQueryConfig.linkedWorkspaces?.length ?? 0) > 0 && Boolean(this.exportQueryConfig.csAssetsUrl); } /** @@ -156,14 +154,8 @@ export class QueryExporter { log.info('Starting export of referenced content types and dependent modules...', this.exportQueryConfig.context); try { - const ctPath = path.join( - sanitizePath(this.exportQueryConfig.exportDir), - 'content_types', - ); - const gfPath = path.join( - sanitizePath(this.exportQueryConfig.exportDir), - 'global_fields', - ); + const ctPath = path.join(sanitizePath(this.exportQueryConfig.exportDir), 'content_types'); + const gfPath = path.join(sanitizePath(this.exportQueryConfig.exportDir), 'global_fields'); const referencedHandler = new ReferencedContentTypesHandler(this.exportQueryConfig); const dependenciesHandler = new ContentTypeDependenciesHandler(this.stackAPIClient, this.exportQueryConfig); @@ -265,7 +257,10 @@ export class QueryExporter { } if (!foundNewCTs && !foundNewGFs) { - log.info('Schema closure complete, no new content types or global fields found', this.exportQueryConfig.context); + log.info( + 'Schema closure complete, no new content types or global fields found', + this.exportQueryConfig.context, + ); break; } } @@ -273,7 +268,10 @@ export class QueryExporter { // Personalize is a single global module exported once after the closure stabilises. await this.moduleExporter.exportModule('personalize'); - log.success('Referenced content types and dependent modules exported successfully', this.exportQueryConfig.context); + log.success( + 'Referenced content types and dependent modules exported successfully', + this.exportQueryConfig.context, + ); } catch (error) { handleAndLogError(error, this.exportQueryConfig.context, 'Error during schema closure expansion'); throw error; @@ -374,11 +372,7 @@ export class QueryExporter { * Export referenced assets into stack assets/ via CMA export module. */ private async exportReferencedStackAssets(assetUIDs: string[]): Promise { - const assetsDir = path.join( - sanitizePath(this.exportQueryConfig.exportDir), - sanitizePath(this.exportQueryConfig.branchName || ''), - 'assets', - ); + const assetsDir = path.join(sanitizePath(this.exportQueryConfig.exportDir), 'assets'); const metadataFilePath = path.join(assetsDir, 'metadata.json'); const assetFilePath = path.join(assetsDir, 'assets.json'); @@ -388,101 +382,101 @@ export class QueryExporter { const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json'); try { - fs.mkdirSync(assetsDir, { recursive: true }); - - // Define batch size - can be configurable through exportQueryConfig - const batchSize = this.exportQueryConfig.assetBatchSize || 100; - - // if asset size is bigger than batch size, then we need to export in batches - // Calculate number of batches - const totalBatches = Math.ceil(assetUIDs.length / batchSize); - log.info(`Processing assets in ${totalBatches} batches of ${batchSize}`, this.exportQueryConfig.context); + fs.mkdirSync(assetsDir, { recursive: true }); + + // Define batch size - can be configurable through exportQueryConfig + const batchSize = this.exportQueryConfig.assetBatchSize || 100; + + // if asset size is bigger than batch size, then we need to export in batches + // Calculate number of batches + const totalBatches = Math.ceil(assetUIDs.length / batchSize); + log.info(`Processing assets in ${totalBatches} batches of ${batchSize}`, this.exportQueryConfig.context); + + // Process assets in batches + for (let i = 0; i < totalBatches; i++) { + const start = i * batchSize; + const end = Math.min(start + batchSize, assetUIDs.length); + const batchAssetUIDs = assetUIDs.slice(start, end); + + log.info( + `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, + this.exportQueryConfig.context, + ); + + const query = { + modules: { + assets: { + uid: { $in: batchAssetUIDs }, + }, + }, + }; + + await this.moduleExporter.exportModule('assets', { query }); + + // Read the current batch's metadata.json and assets.json files + const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath)); + const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath)); + + // Check if this is the first batch + if (i === 0) { + // For first batch, initialize temp files with current content + fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata); + fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets); + log.info(`Initialized temporary files with first batch data`, this.exportQueryConfig.context); + } else { + // For subsequent batches, append to temp files with incremented keys + + // Handle metadata (which contains arrays of asset info) + const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {}; + + // Merge metadata by combining arrays + if (currentMetadata) { + Object.keys(currentMetadata).forEach((key: string) => { + if (!tempMetadata[key]) { + tempMetadata[key] = currentMetadata[key]; + } + }); + } - // Process assets in batches - for (let i = 0; i < totalBatches; i++) { - const start = i * batchSize; - const end = Math.min(start + batchSize, assetUIDs.length); - const batchAssetUIDs = assetUIDs.slice(start, end); + // Write updated metadata back to temp file + fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata); - log.info( - `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, - this.exportQueryConfig.context, - ); + // Handle assets (which is an object with numeric keys) + const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {}; + let nextIndex = Object.keys(tempAssets).length + 1; - const query = { - modules: { - assets: { - uid: { $in: batchAssetUIDs }, - }, - }, - }; - - await this.moduleExporter.exportModule('assets', { query }); - - // Read the current batch's metadata.json and assets.json files - const currentMetadata: any = fsUtil.readFile(sanitizePath(metadataFilePath)); - const currentAssets: any = fsUtil.readFile(sanitizePath(assetFilePath)); - - // Check if this is the first batch - if (i === 0) { - // For first batch, initialize temp files with current content - fsUtil.writeFile(sanitizePath(tempMetadataFilePath), currentMetadata); - fsUtil.writeFile(sanitizePath(tempAssetFilePath), currentAssets); - log.info(`Initialized temporary files with first batch data`, this.exportQueryConfig.context); - } else { - // For subsequent batches, append to temp files with incremented keys - - // Handle metadata (which contains arrays of asset info) - const tempMetadata: any = fsUtil.readFile(sanitizePath(tempMetadataFilePath)) || {}; - - // Merge metadata by combining arrays - if (currentMetadata) { - Object.keys(currentMetadata).forEach((key: string) => { - if (!tempMetadata[key]) { - tempMetadata[key] = currentMetadata[key]; - } - }); - } - - // Write updated metadata back to temp file - fsUtil.writeFile(sanitizePath(tempMetadataFilePath), tempMetadata); - - // Handle assets (which is an object with numeric keys) - const tempAssets: any = fsUtil.readFile(sanitizePath(tempAssetFilePath)) || {}; - let nextIndex = Object.keys(tempAssets).length + 1; - - // Add current assets with incremented keys - Object.values(currentAssets).forEach((value: any) => { - tempAssets[nextIndex.toString()] = value; - nextIndex++; - }); + // Add current assets with incremented keys + Object.values(currentAssets).forEach((value: any) => { + tempAssets[nextIndex.toString()] = value; + nextIndex++; + }); - fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets); + fsUtil.writeFile(sanitizePath(tempAssetFilePath), tempAssets); - log.info(`Updated temporary files with batch ${i + 1} data`, this.exportQueryConfig.context); - } + log.info(`Updated temporary files with batch ${i + 1} data`, this.exportQueryConfig.context); + } - // Optional: Add delay between batches to avoid rate limiting - if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) { - await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs)); - } + // Optional: Add delay between batches to avoid rate limiting + if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) { + await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs)); } + } - // After all batches are processed, copy temp files back to original files - const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath)); - const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath)); + // After all batches are processed, copy temp files back to original files + const finalMetadata = fsUtil.readFile(sanitizePath(tempMetadataFilePath)); + const finalAssets = fsUtil.readFile(sanitizePath(tempAssetFilePath)); - fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata); - fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets); + fsUtil.writeFile(sanitizePath(metadataFilePath), finalMetadata); + fsUtil.writeFile(sanitizePath(assetFilePath), finalAssets); - log.info(`Final data written back to original files`, this.exportQueryConfig.context); + log.info(`Final data written back to original files`, this.exportQueryConfig.context); - // Clean up temp files - fsUtil.removeFile(sanitizePath(tempMetadataFilePath)); - fsUtil.removeFile(sanitizePath(tempAssetFilePath)); + // Clean up temp files + fsUtil.removeFile(sanitizePath(tempMetadataFilePath)); + fsUtil.removeFile(sanitizePath(tempAssetFilePath)); - log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); - log.success('Referenced assets exported successfully', this.exportQueryConfig.context); + log.info(`Temporary files cleaned up`, this.exportQueryConfig.context); + log.success('Referenced assets exported successfully', this.exportQueryConfig.context); } catch (error) { handleAndLogError(error, this.exportQueryConfig.context, 'Error exporting stack referenced assets'); throw error; From 3d82c99259132b052da228458b96158a2c260e0f Mon Sep 17 00:00:00 2001 From: shafeeqd959 Date: Tue, 2 Jun 2026 14:35:20 +0530 Subject: [PATCH 5/5] fixed asset management test cases --- .../import-setup-asset-mappers.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/contentstack-asset-management/test/unit/import-setup/import-setup-asset-mappers.test.ts b/packages/contentstack-asset-management/test/unit/import-setup/import-setup-asset-mappers.test.ts index 936a6a344..b2b74383f 100644 --- a/packages/contentstack-asset-management/test/unit/import-setup/import-setup-asset-mappers.test.ts +++ b/packages/contentstack-asset-management/test/unit/import-setup/import-setup-asset-mappers.test.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { stub, restore } from 'sinon'; -import { AssetManagementAdapter } from '../../../src/utils/asset-management-api-adapter'; +import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; import ImportAssets from '../../../src/import/assets'; import ImportSetupAssetMappers from '../../../src/import-setup/import-setup-asset-mappers'; @@ -77,8 +77,8 @@ describe('ImportSetupAssetMappers', () => { fs.mkdirSync(path.join(contentDir, 'spaces', 'amspace01'), { recursive: true }); fs.mkdirSync(backupDir, { recursive: true }); - stub(AssetManagementAdapter.prototype, 'init').resolves(); - stub(AssetManagementAdapter.prototype, 'listSpaces').resolves({ + stub(CSAssetsAdapter.prototype, 'init').resolves(); + stub(CSAssetsAdapter.prototype, 'listSpaces').resolves({ spaces: [{ uid: 'amspace01' }], }); stub(ImportAssets.prototype, 'buildIdentityMappersFromExport').resolves({ @@ -133,8 +133,8 @@ describe('ImportSetupAssetMappers', () => { fs.mkdirSync(path.join(contentDir, 'spaces', 'amspace01'), { recursive: true }); fs.mkdirSync(backupDir, { recursive: true }); - stub(AssetManagementAdapter.prototype, 'init').resolves(); - stub(AssetManagementAdapter.prototype, 'listSpaces').resolves({ spaces: [] }); + stub(CSAssetsAdapter.prototype, 'init').resolves(); + stub(CSAssetsAdapter.prototype, 'listSpaces').resolves({ spaces: [] }); const buildStub = stub(ImportAssets.prototype, 'buildIdentityMappersFromExport').resolves({ uidMap: {}, @@ -176,8 +176,8 @@ describe('ImportSetupAssetMappers', () => { fs.mkdirSync(path.join(contentDir, 'custom_spaces', 'amspace99'), { recursive: true }); fs.mkdirSync(backupDir, { recursive: true }); - stub(AssetManagementAdapter.prototype, 'init').resolves(); - stub(AssetManagementAdapter.prototype, 'listSpaces').resolves({ + stub(CSAssetsAdapter.prototype, 'init').resolves(); + stub(CSAssetsAdapter.prototype, 'listSpaces').resolves({ spaces: [{ uid: 'amspace99' }], }); @@ -231,8 +231,8 @@ describe('ImportSetupAssetMappers', () => { fs.mkdirSync(path.join(contentDir, 'spaces', 'amX'), { recursive: true }); fs.mkdirSync(backupDir, { recursive: true }); - stub(AssetManagementAdapter.prototype, 'init').resolves(); - stub(AssetManagementAdapter.prototype, 'listSpaces').resolves({ spaces: [{ uid: 'amX' }] }); + stub(CSAssetsAdapter.prototype, 'init').resolves(); + stub(CSAssetsAdapter.prototype, 'listSpaces').resolves({ spaces: [{ uid: 'amX' }] }); stub(ImportAssets.prototype, 'buildIdentityMappersFromExport').callsFake(async function fetchConcCheck( this: ImportAssets,