diff --git a/.talismanrc b/.talismanrc index be53b7c43..e302b6ed9 100644 --- a/.talismanrc +++ b/.talismanrc @@ -19,4 +19,10 @@ fileignoreconfig: checksum: 7b043a59fc9c523d5f772c1b81d6d4b6c65fb7f8edb8df73e48ba821e7298f0b - filename: packages/contentstack-content-type/eslint.config.js checksum: 26da78717a38d8e7464a069626213dd3010efa6e50f91efbc996f26b18346948 +- filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts + checksum: 22708ea1e27a48a5741426a8e17e5d8b243864d877066861bc275d82393002eb +- filename: packages/contentstack-asset-management/src/export/assets.ts + checksum: b169481a31393a9036fbe4d41429bfee3d0f321629f01a72089469ddf5e8826d +- filename: packages/contentstack-asset-management/src/export/assets.ts + checksum: 0a4e04bc91f65cb695a4ca0415dc042b64e6563f0b4a3b718cd9a0ac0d1d7fab version: '1.0' diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index ec288b72a..65adc0541 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -13,7 +13,6 @@ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ 'created_by', 'updated_at', 'updated_by', - 'is_system', 'asset_types_count', ] as const; export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ @@ -21,7 +20,6 @@ export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ 'created_by', 'updated_at', 'updated_by', - 'is_system', 'category', 'preview_image_url', 'category_detail', diff --git a/packages/contentstack-asset-management/src/export/asset-types.ts b/packages/contentstack-asset-management/src/export/asset-types.ts index 7b171a25f..f9b859e71 100644 --- a/packages/contentstack-asset-management/src/export/asset-types.ts +++ b/packages/contentstack-asset-management/src/export/asset-types.ts @@ -13,19 +13,15 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter { super(apiConfig, exportContext); } - async start(spaceUid: string): Promise { + async start(spaceUid: string): Promise { await this.init(); log.debug('Starting shared asset types export process...', this.exportContext.context); + log.info('Exporting shared asset types...', this.exportContext.context); const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid, this.apiPageSize, this.apiFetchConcurrency); const items = getArrayFromResponse(assetTypesData, 'asset_types'); const dir = this.getAssetTypesDir(); - if (items.length === 0) { - log.info('No asset types to export, writing empty asset-types', this.exportContext.context); - } else { - log.debug(`Writing ${items.length} shared asset types`, this.exportContext.context); - } await this.writeItemsToChunkedJson( dir, 'asset-types.json', @@ -33,6 +29,13 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter { ['uid', 'title', 'category', 'file_extension'], items, ); + log.info( + items.length === 0 + ? 'No asset types to export' + : `Exported ${items.length} shared asset type(s)`, + this.exportContext.context, + ); this.tick(true, `asset_types (${items.length})`, null); + return items.length; } } diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 8fdad2039..662da831c 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -7,7 +7,7 @@ import { configHandler, log, FsUtility } from '@contentstack/cli-utilities'; import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsExportAdapter } from './base'; -import { writeStreamToFile } from '../utils/export-helpers'; +import { writeStreamToFile, getArrayFromResponse } from '../utils/export-helpers'; import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry'; import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter'; @@ -17,6 +17,9 @@ const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid']; type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }; +/** Per-space export counts surfaced to the summary (assets = downloaded binaries; folders = entities). */ +export type SpaceExportCounts = { assets: number; folders: number }; + export default class ExportAssets extends CSAssetsExportAdapter { constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) { super(apiConfig, exportContext); @@ -26,7 +29,7 @@ export default class ExportAssets extends CSAssetsExportAdapter { return Boolean(asset?.url && (asset?.uid ?? asset?._uid)); } - async start(workspace: LinkedWorkspace, spaceDir: string): Promise { + async start(workspace: LinkedWorkspace, spaceDir: string): Promise { await this.init(); log.debug(`Starting assets export for space ${workspace.space_uid}`, this.exportContext.context); @@ -75,14 +78,17 @@ export default class ExportAssets extends CSAssetsExportAdapter { this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null); log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context); - await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount); + const assetsDownloaded = await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount); + + const folderCount = getArrayFromResponse(folders, 'folders').length; + return { assets: assetsDownloaded, folders: folderCount }; } /** * Download asset binaries by reading the just-written chunked `assets.json` back from disk * (one chunk at a time), so we never re-materialize the whole asset list in memory. */ - private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise { + private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise { const filesDir = pResolve(assetsDir, 'files'); await mkdir(filesDir, { recursive: true }); @@ -105,6 +111,27 @@ export default class ExportAssets extends CSAssetsExportAdapter { chunkReadLogLabel: 'assets', onOpenError: (err) => log.debug(`Could not open assets.json for download: ${err}`, this.exportContext.context), onEmptyIndexer: () => log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context), + // A chunk that fails to read back would otherwise drop its downloads silently. `records` are + // recovered from metadata.json, so we count + surface each lost asset by identity here — no + // separate full-metadata reconcile (which would re-materialize the whole set every run). + onChunkError: (records, err) => { + log.error( + `Failed to read an asset chunk back from disk during download for space ${spaceUid}: ${ + (err as Error)?.message ?? String(err) + }`, + this.exportContext.context, + ); + for (const rec of records) { + if (!this.isDownloadable(rec)) continue; + downloadFail += 1; + const label = rec.file_name ?? rec.filename ?? rec.uid ?? 'asset'; + this.tick(false, `asset: ${label}`, 'Asset chunk unreadable'); + log.error( + `Asset ${rec.uid ?? ''} not downloaded — chunk unreadable for space ${spaceUid}`, + this.exportContext.context, + ); + } + }, }, async (records) => { const valid = records.filter((asset) => this.isDownloadable(asset)); @@ -153,7 +180,10 @@ export default class ExportAssets extends CSAssetsExportAdapter { downloadFail += 1; const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; this.tick(false, `asset: ${filename}`, err); - log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); + log.error( + `Failed to download asset ${uid} (${filename}): ${(e as Error)?.message ?? String(e)}`, + this.exportContext.context, + ); } }; @@ -161,18 +191,6 @@ export default class ExportAssets extends CSAssetsExportAdapter { }, ); - // Completeness check: a chunk that fails to read back is skipped (logged at debug) by - // forEachChunkedJsonStore, which would silently drop those downloads. Reconcile attempts - // (ok + failed) against what streaming counted as downloadable. - const attempted = downloadOk + downloadFail; - if (attempted < expectedDownloads) { - log.warn( - `Asset downloads for space ${spaceUid} incomplete: expected ${expectedDownloads}, attempted ${attempted}` + - ` — ${expectedDownloads - attempted} asset(s) were never read back for download.`, - this.exportContext.context, - ); - } - log.info( downloadFail === 0 ? `Finished downloading ${downloadOk} asset file(s) for space ${spaceUid}` @@ -180,8 +198,11 @@ export default class ExportAssets extends CSAssetsExportAdapter { this.exportContext.context, ); log.debug( - `Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}`, + `Asset downloads finished for space ${spaceUid}: ok=${downloadOk}, failed=${downloadFail}, expected=${expectedDownloads}`, this.exportContext.context, ); + + return downloadOk; } + } diff --git a/packages/contentstack-asset-management/src/export/fields.ts b/packages/contentstack-asset-management/src/export/fields.ts index a4cfe8460..fbee72690 100644 --- a/packages/contentstack-asset-management/src/export/fields.ts +++ b/packages/contentstack-asset-management/src/export/fields.ts @@ -13,20 +13,21 @@ export default class ExportFields extends CSAssetsExportAdapter { super(apiConfig, exportContext); } - async start(spaceUid: string): Promise { + async start(spaceUid: string): Promise { await this.init(); log.debug('Starting shared fields export process...', this.exportContext.context); + log.info('Exporting shared fields...', this.exportContext.context); const fieldsData = await this.getWorkspaceFields(spaceUid, this.apiPageSize, this.apiFetchConcurrency); const items = getArrayFromResponse(fieldsData, 'fields'); const dir = this.getFieldsDir(); - if (items.length === 0) { - log.info('No field items to export, writing empty fields', this.exportContext.context); - } else { - log.debug(`Writing ${items.length} shared fields`, this.exportContext.context); - } await this.writeItemsToChunkedJson(dir, 'fields.json', 'fields', ['uid', 'title', 'display_type'], items); + log.info( + items.length === 0 ? 'No fields to export' : `Exported ${items.length} shared field(s)`, + this.exportContext.context, + ); this.tick(true, `fields (${items.length})`, null); + return items.length; } } diff --git a/packages/contentstack-asset-management/src/export/index.ts b/packages/contentstack-asset-management/src/export/index.ts index 14727ed70..4d07a46a6 100644 --- a/packages/contentstack-asset-management/src/export/index.ts +++ b/packages/contentstack-asset-management/src/export/index.ts @@ -1,4 +1,5 @@ export { ExportSpaces, exportSpaceStructure } from './spaces'; +export type { AssetExportCounts } from './spaces'; export { default as ExportAssetTypes } from './asset-types'; export { default as ExportFields } from './fields'; export { default as ExportAssets } from './assets'; diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index 6d3b0ab13..c41070158 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -9,6 +9,17 @@ import ExportAssetTypes from './asset-types'; import ExportFields from './fields'; import ExportWorkspace from './workspaces'; +/** + * Real entity counts for the export summary (Bug 3 — "everything under ASSETS"): + * assets = downloaded binaries, folders = folder entities, plus shared asset_types and fields. + */ +export type AssetExportCounts = { + assets: number; + folders: number; + assetTypes: number; + fields: number; +}; + /** * Orchestrates the full Contentstack Assets export: shared asset types and fields, * then per-workspace metadata and assets (including internal download). @@ -27,7 +38,7 @@ export class ExportSpaces { this.parentProgressManager = parent; } - async start(): Promise { + async start(): Promise { const { linkedWorkspaces, exportDir, @@ -42,7 +53,7 @@ export class ExportSpaces { if (!linkedWorkspaces.length) { log.debug('No linked workspaces to export', context); - return; + return { assets: 0, folders: 0, assetTypes: 0, fields: 0 }; } log.debug('Starting Contentstack Assets export process...', context); @@ -91,6 +102,11 @@ export class ExportSpaces { const firstSpaceUid = linkedWorkspaces[0].space_uid; let bootstrapFailed = false; let anySpaceFailed = false; + // Real entity counts accumulated for the summary (Bug 3). + let assetsTotal = 0; + let foldersTotal = 0; + let assetTypesCount = 0; + let fieldsCount = 0; try { progress.startProcess(PROCESS_NAMES.AM_FIELDS); progress.startProcess(PROCESS_NAMES.AM_ASSET_TYPES); @@ -100,7 +116,10 @@ export class ExportSpaces { const exportFields = new ExportFields(apiConfig, exportContext); exportFields.setParentProgressManager(progress); try { - await Promise.all([exportAssetTypes.start(firstSpaceUid), exportFields.start(firstSpaceUid)]); + [assetTypesCount, fieldsCount] = await Promise.all([ + exportAssetTypes.start(firstSpaceUid), + exportFields.start(firstSpaceUid), + ]); progress.completeProcess(PROCESS_NAMES.AM_FIELDS, true); progress.completeProcess(PROCESS_NAMES.AM_ASSET_TYPES, true); } catch (bootstrapErr) { @@ -118,7 +137,9 @@ export class ExportSpaces { try { const exportWorkspace = new ExportWorkspace(apiConfig, exportContext); exportWorkspace.setParentProgressManager(progress); - await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess); + const spaceCounts = await exportWorkspace.start(ws, spaceDir, branchName || 'main', spaceProcess); + assetsTotal += spaceCounts.assets; + foldersTotal += spaceCounts.folders; progress.completeProcess(spaceProcess, true); log.debug(`Exported workspace structure for space ${ws.space_uid}`, context); } catch (err) { @@ -142,6 +163,8 @@ export class ExportSpaces { context, ); log.debug('Contentstack Assets export completed', context); + + return { assets: assetsTotal, folders: foldersTotal, assetTypes: assetTypesCount, fields: fieldsCount }; } catch (err) { if (!bootstrapFailed) { // Mark any spaces that hadn't been processed as failed so the multibar @@ -170,6 +193,6 @@ export class ExportSpaces { /** * Entry point for callers that prefer a function. Delegates to ExportSpaces. */ -export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise { - await new ExportSpaces(options).start(); +export async function exportSpaceStructure(options: AssetManagementExportOptions): Promise { + return new ExportSpaces(options).start(); } diff --git a/packages/contentstack-asset-management/src/export/workspaces.ts b/packages/contentstack-asset-management/src/export/workspaces.ts index 14dd5c1a5..328399e9a 100644 --- a/packages/contentstack-asset-management/src/export/workspaces.ts +++ b/packages/contentstack-asset-management/src/export/workspaces.ts @@ -5,7 +5,7 @@ import { log } from '@contentstack/cli-utilities'; import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsExportAdapter } from './base'; -import ExportAssets from './assets'; +import ExportAssets, { type SpaceExportCounts } from './assets'; export default class ExportWorkspace extends CSAssetsExportAdapter { constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) { @@ -26,7 +26,7 @@ export default class ExportWorkspace extends CSAssetsExportAdapter { spaceDir: string, branchName: string, spaceProcessName?: string, - ): Promise { + ): Promise { await this.init(); if (spaceProcessName) { @@ -59,7 +59,8 @@ export default class ExportWorkspace extends CSAssetsExportAdapter { if (spaceProcessName) { assetsExporter.setProcessName(spaceProcessName); } - await assetsExporter.start(workspace, spaceDir); + const counts = await assetsExporter.start(workspace, spaceDir); log.debug(`Exported workspace structure for space ${workspace.space_uid}`, this.exportContext.context); + return counts; } } diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts index dfc997512..8bfb1236a 100644 --- a/packages/contentstack-asset-management/src/import/asset-types.ts +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -138,7 +138,7 @@ export default class ImportAssetTypes extends CSAssetsImportAdapter { log.debug(`Imported asset type: ${uid}`, this.importContext.context); } catch (e) { this.failureCount += 1; - log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); + log.error(`Failed to import asset type ${uid}: ${(e as Error)?.message ?? String(e)}`, this.importContext.context); } }); } diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts index 5b1187410..a55476cf9 100644 --- a/packages/contentstack-asset-management/src/import/assets.ts +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -196,7 +196,17 @@ export default class ImportAssets extends CSAssetsImportAdapter { await forEachChunkRecordsFromFs( assetFs, - { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + { + context: this.importContext.context, + chunkReadLogLabel: 'assets', + onChunkError: (_records, err) => + log.error( + `Failed to read an asset chunk back from disk during import for space ${newSpaceUid}: ${ + (err as Error)?.message ?? String(err) + }`, + this.importContext.context, + ), + }, async (assetChunk) => { exportRowCount += assetChunk.length; const uploadJobs: UploadJob[] = []; @@ -208,8 +218,8 @@ export default class ImportAssets extends CSAssetsImportAdapter { if (!existsSync(filePath)) { missingFiles += 1; - log.warn(`Asset file not found: ${filePath}, skipping`, this.importContext.context); this.tick(false, `asset: ${oldUid}`, 'File not found on disk'); + log.error(`Asset file not found: ${filePath}`, this.importContext.context); continue; } @@ -236,13 +246,8 @@ export default class ImportAssets extends CSAssetsImportAdapter { description: asset.description, parent_uid: mappedParentUid, }); - uidMap[oldUid] = created.uid; - - if (asset.url && created.url) { - urlMap[asset.url] = created.url; - } - + if (asset.url && created.url) urlMap[asset.url] = created.url; this.tick(true, `asset: ${filename}`, null); uploadOk += 1; log.debug(`Uploaded asset ${oldUid} → ${created.uid} (${filePath})`, this.importContext.context); @@ -253,7 +258,10 @@ export default class ImportAssets extends CSAssetsImportAdapter { `asset: ${filename}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, ); - log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + log.error( + `Failed to upload asset ${oldUid}: ${(e as Error)?.message ?? String(e)}`, + this.importContext.context, + ); } }, ); @@ -367,7 +375,10 @@ export default class ImportAssets extends CSAssetsImportAdapter { `folder: ${folder.title}`, (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].FAILED, ); - log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); + log.error( + `Failed to create folder ${folder.uid}: ${(e as Error)?.message ?? String(e)}`, + this.importContext.context, + ); } }); diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts index cf0747598..23502e881 100644 --- a/packages/contentstack-asset-management/src/import/fields.ts +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -140,7 +140,7 @@ export default class ImportFields extends CSAssetsImportAdapter { log.debug(`Imported field: ${uid}`, this.importContext.context); } catch (e) { this.failureCount += 1; - log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); + log.error(`Failed to import field ${uid}: ${(e as Error)?.message ?? String(e)}`, this.importContext.context); } }); } diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts index 838cfe653..18bcfdbeb 100644 --- a/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts +++ b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts @@ -1,16 +1,24 @@ import { FsUtility, log } from '@contentstack/cli-utilities'; -export type ForEachChunkedJsonStoreOptions = { +export type ForEachChunkedJsonStoreOptions = { context?: Record; /** Shown in log.debug: `Error reading