Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .talismanrc
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,10 @@ fileignoreconfig:
checksum: 7b043a59fc9c523d5f772c1b81d6d4b6c65fb7f8edb8df73e48ba821e7298f0b
- filename: packages/contentstack-content-type/eslint.config.js
checksum: 26da78717a38d8e7464a069626213dd3010efa6e50f91efbc996f26b18346948
- filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts
checksum: 22708ea1e27a48a5741426a8e17e5d8b243864d877066861bc275d82393002eb
- filename: packages/contentstack-asset-management/src/export/assets.ts
checksum: b169481a31393a9036fbe4d41429bfee3d0f321629f01a72089469ddf5e8826d
- filename: packages/contentstack-asset-management/src/export/assets.ts
checksum: 0a4e04bc91f65cb695a4ca0415dc042b64e6563f0b4a3b718cd9a0ac0d1d7fab
version: '1.0'
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,13 @@ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [
'created_by',
'updated_at',
'updated_by',
'is_system',
'asset_types_count',
] as const;
export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [
'created_at',
'created_by',
'updated_at',
'updated_by',
'is_system',
'category',
'preview_image_url',
'category_detail',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,26 +13,29 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter {
super(apiConfig, exportContext);
}

async start(spaceUid: string): Promise<void> {
async start(spaceUid: string): Promise<number> {
await this.init();

log.debug('Starting shared asset types export process...', this.exportContext.context);
log.info('Exporting shared asset types...', this.exportContext.context);

const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid, this.apiPageSize, this.apiFetchConcurrency);
const items = getArrayFromResponse(assetTypesData, 'asset_types');
const dir = this.getAssetTypesDir();
if (items.length === 0) {
log.info('No asset types to export, writing empty asset-types', this.exportContext.context);
} else {
log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(
dir,
'asset-types.json',
'asset_types',
['uid', 'title', 'category', 'file_extension'],
items,
);
log.info(
items.length === 0
? 'No asset types to export'
: `Exported ${items.length} shared asset type(s)`,
this.exportContext.context,
);
this.tick(true, `asset_types (${items.length})`, null);
return items.length;
}
}
57 changes: 39 additions & 18 deletions packages/contentstack-asset-management/src/export/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { configHandler, log, FsUtility } from '@contentstack/cli-utilities';
import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
import type { ExportContext } from '../types/export-types';
import { CSAssetsExportAdapter } from './base';
import { writeStreamToFile } from '../utils/export-helpers';
import { writeStreamToFile, getArrayFromResponse } from '../utils/export-helpers';
import { forEachChunkedJsonStore } from '../utils/chunked-json-reader';
import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry';
import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter';
Expand All @@ -17,6 +17,9 @@ const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid'];

type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string };

/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */
export type SpaceExportCounts = { assets: number; folders: number };

export default class ExportAssets extends CSAssetsExportAdapter {
constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
super(apiConfig, exportContext);
Expand All @@ -26,7 +29,7 @@ export default class ExportAssets extends CSAssetsExportAdapter {
return Boolean(asset?.url && (asset?.uid ?? asset?._uid));
}

async start(workspace: LinkedWorkspace, spaceDir: string): Promise<void> {
async start(workspace: LinkedWorkspace, spaceDir: string): Promise<SpaceExportCounts> {
await this.init();

log.debug(`Starting assets export for space ${workspace.space_uid}`, this.exportContext.context);
Expand Down Expand Up @@ -75,14 +78,17 @@ export default class ExportAssets extends CSAssetsExportAdapter {
this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null);

log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context);
await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);
const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount);

const folderCount = getArrayFromResponse(folders, 'folders').length;
return { assets: assetsDownloaded, folders: folderCount };
}

/**
* Download asset binaries by reading the just-written chunked `assets.json` back from disk
* (one chunk at a time), so we never re-materialize the whole asset list in memory.
*/
private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<void> {
private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise<number> {
const filesDir = pResolve(assetsDir, 'files');
await mkdir(filesDir, { recursive: true });

Expand All @@ -105,6 +111,27 @@ export default class ExportAssets extends CSAssetsExportAdapter {
chunkReadLogLabel: 'assets',
onOpenError: (err) => log.debug(`Could not open assets.json for download: ${err}`, this.exportContext.context),
onEmptyIndexer: () => log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context),
// A chunk that fails to read back would otherwise drop its downloads silently. `records` are
// recovered from metadata.json, so we count + surface each lost asset by identity here — no
// separate full-metadata reconcile (which would re-materialize the whole set every run).
onChunkError: (records, err) => {
log.error(
`Failed to read an asset chunk back from disk during download for space ${spaceUid}: ${
(err as Error)?.message ?? String(err)
}`,
this.exportContext.context,
);
for (const rec of records) {
if (!this.isDownloadable(rec)) continue;
downloadFail += 1;
const label = rec.file_name ?? rec.filename ?? rec.uid ?? 'asset';
this.tick(false, `asset: ${label}`, 'Asset chunk unreadable');
log.error(
`Asset ${rec.uid ?? '<unknown>'} not downloaded — chunk unreadable for space ${spaceUid}`,
this.exportContext.context,
);
}
},
},
async (records) => {
const valid = records.filter((asset) => this.isDownloadable(asset));
Expand Down Expand Up @@ -153,35 +180,29 @@ export default class ExportAssets extends CSAssetsExportAdapter {
downloadFail += 1;
const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED;
this.tick(false, `asset: ${filename}`, err);
log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context);
log.error(
`Failed to download asset ${uid} (${filename}): ${(e as Error)?.message ?? String(e)}`,
this.exportContext.context,
);
}
};

await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler);
},
);

// Completeness check: a chunk that fails to read back is skipped (logged at debug) by
// forEachChunkedJsonStore, which would silently drop those downloads. Reconcile attempts
// (ok + failed) against what streaming counted as downloadable.
const attempted = downloadOk + downloadFail;
if (attempted < expectedDownloads) {
log.warn(
`Asset downloads for space ${spaceUid} incomplete: expected ${expectedDownloads}, attempted ${attempted}` +
` — ${expectedDownloads - attempted} asset(s) were never read back for download.`,
this.exportContext.context,
);
}

log.info(
downloadFail === 0
? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}`
: `Asset downloads for space ${spaceUid} completed with errors: ${downloadOk} succeeded, ${downloadFail} failed`,
this.exportContext.context,
);
log.debug(
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`,
`Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, expected=${expectedDownloads}`,
this.exportContext.context,
);

return downloadOk;
}

}
13 changes: 7 additions & 6 deletions packages/contentstack-asset-management/src/export/fields.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,21 @@ export default class ExportFields extends CSAssetsExportAdapter {
super(apiConfig, exportContext);
}

async start(spaceUid: string): Promise<void> {
async start(spaceUid: string): Promise<number> {
await this.init();

log.debug('Starting shared fields export process...', this.exportContext.context);
log.info('Exporting shared fields...', this.exportContext.context);

const fieldsData = await this.getWorkspaceFields(spaceUid, this.apiPageSize, this.apiFetchConcurrency);
const items = getArrayFromResponse(fieldsData, 'fields');
const dir = this.getFieldsDir();
if (items.length === 0) {
log.info('No field items to export, writing empty fields', this.exportContext.context);
} else {
log.debug(`Writing ${items.length} shared fields`, this.exportContext.context);
}
await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items);
log.info(
items.length === 0 ? 'No fields to export' : `Exported ${items.length} shared field(s)`,
this.exportContext.context,
);
this.tick(true, `fields (${items.length})`, null);
return items.length;
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export { ExportSpaces, exportSpaceStructure } from './spaces';
export type { AssetExportCounts } from './spaces';
export { default as ExportAssetTypes } from './asset-types';
export { default as ExportFields } from './fields';
export { default as ExportAssets } from './assets';
Expand Down
35 changes: 29 additions & 6 deletions packages/contentstack-asset-management/src/export/spaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@ import ExportAssetTypes from './asset-types';
import ExportFields from './fields';
import ExportWorkspace from './workspaces';

/**
* Real entity counts for the export summary (Bug 3 — "everything under ASSETS"):
* assets = downloaded binaries, folders = folder entities, plus shared asset_types and fields.
*/
export type AssetExportCounts = {
assets: number;
folders: number;
assetTypes: number;
fields: number;
};

/**
* Orchestrates the full Contentstack Assets export: shared asset types and fields,
* then per-workspace metadata and assets (including internal download).
Expand All @@ -27,7 +38,7 @@ export class ExportSpaces {
this.parentProgressManager = parent;
}

async start(): Promise<void> {
async start(): Promise<AssetExportCounts> {
const {
linkedWorkspaces,
exportDir,
Expand All @@ -42,7 +53,7 @@ export class ExportSpaces {

if (!linkedWorkspaces.length) {
log.debug('No linked workspaces to export', context);
return;
return { assets: 0, folders: 0, assetTypes: 0, fields: 0 };
}

log.debug('Starting Contentstack Assets export process...', context);
Expand Down Expand Up @@ -91,6 +102,11 @@ export class ExportSpaces {
const firstSpaceUid = linkedWorkspaces[0].space_uid;
let bootstrapFailed = false;
let anySpaceFailed = false;
// Real entity counts accumulated for the summary (Bug 3).
let assetsTotal = 0;
let foldersTotal = 0;
let assetTypesCount = 0;
let fieldsCount = 0;
try {
progress.startProcess(PROCESS_NAMES.AM_FIELDS);
progress.startProcess(PROCESS_NAMES.AM_ASSET_TYPES);
Expand All @@ -100,7 +116,10 @@ export class ExportSpaces {
const exportFields = new ExportFields(apiConfig, exportContext);
exportFields.setParentProgressManager(progress);
try {
await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]);
[assetTypesCount, fieldsCount] = await Promise.all([
exportAssetTypes.start(firstSpaceUid),
exportFields.start(firstSpaceUid),
]);
progress.completeProcess(PROCESS_NAMES.AM_FIELDS, true);
progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, true);
} catch (bootstrapErr) {
Expand All @@ -118,7 +137,9 @@ export class ExportSpaces {
try {
const exportWorkspace = new ExportWorkspace(apiConfig, exportContext);
exportWorkspace.setParentProgressManager(progress);
await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess);
assetsTotal += spaceCounts.assets;
foldersTotal += spaceCounts.folders;
progress.completeProcess(spaceProcess, true);
log.debug(`Exported workspace structure for space ${ws.space_uid}`, context);
} catch (err) {
Expand All @@ -142,6 +163,8 @@ export class ExportSpaces {
context,
);
log.debug('Contentstack Assets export completed', context);

return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount };
} catch (err) {
if (!bootstrapFailed) {
// Mark any spaces that hadn't been processed as failed so the multibar
Expand Down Expand Up @@ -170,6 +193,6 @@ export class ExportSpaces {
/**
* Entry point for callers that prefer a function. Delegates to ExportSpaces.
*/
export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise<void> {
await new ExportSpaces(options).start();
export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise<AssetExportCounts> {
return new ExportSpaces(options).start();
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { log } from '@contentstack/cli-utilities';
import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api';
import type { ExportContext } from '../types/export-types';
import { CSAssetsExportAdapter } from './base';
import ExportAssets from './assets';
import ExportAssets, { type SpaceExportCounts } from './assets';

export default class ExportWorkspace extends CSAssetsExportAdapter {
constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) {
Expand All @@ -26,7 +26,7 @@ export default class ExportWorkspace extends CSAssetsExportAdapter {
spaceDir: string,
branchName: string,
spaceProcessName?: string,
): Promise<void> {
): Promise<SpaceExportCounts> {
await this.init();

if (spaceProcessName) {
Expand Down Expand Up @@ -59,7 +59,8 @@ export default class ExportWorkspace extends CSAssetsExportAdapter {
if (spaceProcessName) {
assetsExporter.setProcessName(spaceProcessName);
}
await assetsExporter.start(workspace, spaceDir);
const counts = await assetsExporter.start(workspace, spaceDir);
log.debug(`Exported workspace structure for space ${workspace.space_uid}`, this.exportContext.context);
return counts;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ export default class ImportAssetTypes extends CSAssetsImportAdapter {
log.debug(`Imported asset type: ${uid}`, this.importContext.context);
} catch (e) {
this.failureCount += 1;
log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context);
log.error(`Failed to import asset type ${uid}: ${(e as Error)?.message ?? String(e)}`, this.importContext.context);
}
});
}
Expand Down
Loading
Loading