diff --git a/.github/workflows/release-v2-beta-plugins.yml b/.github/workflows/release-v2-beta-plugins.yml index 58c10ac87..c79c4277d 100644 --- a/.github/workflows/release-v2-beta-plugins.yml +++ b/.github/workflows/release-v2-beta-plugins.yml @@ -135,6 +135,14 @@ jobs: package: ./packages/contentstack-branches/package.json tag: beta + # Apps CLI + - name: Publishing apps-cli (Production) + uses: JS-DevTools/npm-publish@v3 + with: + token: ${{ secrets.NPM_TOKEN }} + package: ./packages/contentstack-apps-cli/package.json + tag: latest + # Query Export - name: Publishing query-export (Beta) uses: JS-DevTools/npm-publish@v3 @@ -183,6 +191,23 @@ jobs: package: ./packages/contentstack-migrate-rte/package.json access: public tag: beta + # External Migrate + - name: Publishing external-migrate (Beta) + uses: JS-DevTools/npm-publish@v3 + with: + token: ${{ secrets.NPM_TOKEN }} + package: ./packages/contentstack-external-migrate/package.json + access: public + tag: beta + + - name: Create Production Release + id: create_release + env: + GITHUB_TOKEN: ${{ secrets.PKG_TOKEN }} + VERSION: ${{ steps.publish-plugins.outputs.version }} + run: | + # Get the previous production release for comparison + PREVIOUS_PROD=$(gh release list --limit 10 | grep -v 'prerelease' | head -1 | cut -f1) # Bulk Operations - name: Publishing bulk-operations (Beta) diff --git a/.talismanrc b/.talismanrc index 789dc0cda..78e4e1bb2 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,4 +1,4 @@ fileignoreconfig: - filename: pnpm-lock.yaml - checksum: cdead0797199d22bbc55b9e5b6b86983f28eb760fabe5e1f2d5139c4456a9131 -version: '1.0' + checksum: 0feb3713a8f2e4a8a1f5f528218c2c578265dc5b31ff283a283fefc949bbafd2 +version: "" \ No newline at end of file diff --git a/package.json b/package.json index 4ae6360da..bfc835efa 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "pnpm": "^10.28.0" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" }, "private": true, "scripts": { diff --git a/packages/contentstack-apps-cli/.eslintignore b/packages/contentstack-apps-cli/.eslintignore deleted file mode 100644 index 9b1c8b133..000000000 --- a/packages/contentstack-apps-cli/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -/dist diff --git a/packages/contentstack-apps-cli/.eslintrc b/packages/contentstack-apps-cli/.eslintrc deleted file mode 100644 index aa58bce72..000000000 --- a/packages/contentstack-apps-cli/.eslintrc +++ /dev/null @@ -1,42 +0,0 @@ -{ - "env": { - "node": true - }, - "parser": "@typescript-eslint/parser", - "parserOptions": { - "project": "tsconfig.json", - "sourceType": "module" - }, - "plugins": [ - "@typescript-eslint" - ], - "extends": [ - "plugin:@typescript-eslint/recommended" - ], - "ignorePatterns": [ - "lib/**/*", - "test/**/*" - ], - "rules": { - "@typescript-eslint/no-unused-vars": [ - "error", - { - "args": "none" - } - ], - "@typescript-eslint/prefer-namespace-keyword": "error", - "quotes": "off", - "semi": "off", - "@typescript-eslint/no-redeclare": "off", - "eqeqeq": [ - "error", - "smart" - ], - "id-match": "error", - "no-eval": "error", - "no-var": "error", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-require-imports": "off", - "prefer-const": "error" - } -} \ No newline at end of file diff --git a/packages/contentstack-apps-cli/eslint.config.js b/packages/contentstack-apps-cli/eslint.config.js new file mode 100644 index 000000000..cb01e63b9 --- /dev/null +++ b/packages/contentstack-apps-cli/eslint.config.js @@ -0,0 +1,47 @@ +import tseslint from 'typescript-eslint'; +import globals from 'globals'; + +export default [ + ...tseslint.configs.recommended, + { + ignores: [ + 'lib/**/*', + 'test/**/*', + 'dist/**/*', + ], + }, + { + languageOptions: { + parser: tseslint.parser, + parserOptions: { + project: './tsconfig.json', + }, + sourceType: 'module', + globals: { + ...globals.node, + }, + }, + plugins: { + '@typescript-eslint': tseslint.plugin, + }, + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { + args: 'none', + }, + ], + '@typescript-eslint/prefer-namespace-keyword': 'error', + quotes: 'off', + semi: 'off', + '@typescript-eslint/no-redeclare': 'off', + eqeqeq: ['error', 'smart'], + 'id-match': 'error', + 'no-eval': 'error', + 'no-var': 'error', + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-require-imports': 'off', + 'prefer-const': 'error', + }, + }, +]; diff --git a/packages/contentstack-apps-cli/package.json b/packages/contentstack-apps-cli/package.json index 79879833c..d6542e672 100644 --- a/packages/contentstack-apps-cli/package.json +++ b/packages/contentstack-apps-cli/package.json @@ -1,6 +1,6 @@ { "name": "@contentstack/apps-cli", - "version": "2.0.0-beta.0", + "version": "2.0.0-beta.1", "description": "App ClI", "author": "Contentstack CLI", "homepage": "https://github.com/contentstack/cli-plugins/tree/main/packages/contentstack-apps-cli", @@ -22,9 +22,9 @@ ], "dependencies": { "@apollo/client": "^3.14.1", - "@contentstack/cli-command": "~2.0.0-beta.8", + "@contentstack/cli-command": "~2.0.0-beta.9", "@contentstack/cli-launch": "^1.10.0", - "@contentstack/cli-utilities": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", "adm-zip": "^0.5.17", "chalk": "^5.6.2", "lodash": "^4.18.1", @@ -46,7 +46,7 @@ "axios": "^1.16.1", "chai": "^4.5.0", "dotenv": "^16.6.1", - "eslint": "^8.57.1", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.157", "eslint-config-oclif-typescript": "^3.1.14", "mocha": "^10.8.2", @@ -80,7 +80,7 @@ }, "scripts": { "build": "npm run clean && shx rm -rf lib && tsc -b", - "lint": "eslint . --ext .ts --config .eslintrc", + "lint": "eslint .", "postpack": "shx rm -f oclif.manifest.json", "posttest": "npm run lint", "prepack": "npm run build && oclif manifest && oclif readme", @@ -91,7 +91,7 @@ "test:unit:report:json": "mocha --reporter json --reporter-options output=report.json --forbid-only \"test/unit/**/*.test.ts\" && nyc --reporter=clover --extension .ts mocha --forbid-only \"test/unit/**/*.test.ts\"" }, "engines": { - "node": ">=16" + "node": ">=22.0.0" }, "bugs": "https://github.com/contentstack/cli-plugins/issues", "keywords": [ diff --git a/packages/contentstack-apps-cli/src/util/common-utils.ts b/packages/contentstack-apps-cli/src/util/common-utils.ts index f72dcb1c4..b9f68def4 100644 --- a/packages/contentstack-apps-cli/src/util/common-utils.ts +++ b/packages/contentstack-apps-cli/src/util/common-utils.ts @@ -120,7 +120,7 @@ function fetchAppInstallations( flags: FlagInput, orgUid: string, options: MarketPlaceOptions -) { +): Promise { const { marketplaceSdk } = options; const app: any = flags["app-uid"]; return marketplaceSdk @@ -218,7 +218,7 @@ function uninstallApp( orgUid: string, options: MarketPlaceOptions, installationUid: string -) { +): Promise { const { marketplaceSdk } = options; // const app: any = flags["app-uid"]; return marketplaceSdk diff --git a/packages/contentstack-asset-management/.eslintrc b/packages/contentstack-asset-management/.eslintrc deleted file mode 100644 index 4d2aef653..000000000 --- a/packages/contentstack-asset-management/.eslintrc +++ /dev/null @@ -1,54 +0,0 @@ -{ - "env": { - "node": true - }, - "parser": "@typescript-eslint/parser", - "parserOptions": { - "project": "tsconfig.json", - "sourceType": "module" - }, - "extends": [ - "oclif-typescript", - "plugin:@typescript-eslint/recommended" - ], - "rules": { - "@typescript-eslint/no-unused-vars": [ - "error", - { - "args": "none" - } - ], - "@typescript-eslint/prefer-namespace-keyword": "error", - "@typescript-eslint/quotes": [ - "error", - "single", - { - "avoidEscape": true, - "allowTemplateLiterals": true - } - ], - "semi": "off", - "@typescript-eslint/type-annotation-spacing": "error", - "@typescript-eslint/no-redeclare": "off", - "eqeqeq": [ - "error", - "smart" - ], - "id-match": "error", - "no-eval": "error", - "no-var": "error", - "quotes": "off", - "indent": "off", - "camelcase": "off", - "comma-dangle": "off", - "arrow-parens": "off", - "operator-linebreak": "off", - "object-curly-spacing": "off", - "node/no-missing-import": "off", - "padding-line-between-statements": "off", - "@typescript-eslint/ban-ts-ignore": "off", - "unicorn/no-abusive-eslint-disable": "off", - "unicorn/consistent-function-scoping": "off", - "@typescript-eslint/no-use-before-define": "off" - } -} diff --git a/packages/contentstack-asset-management/README.md b/packages/contentstack-asset-management/README.md new file mode 100644 index 000000000..955ec5e2c --- /dev/null +++ b/packages/contentstack-asset-management/README.md @@ -0,0 +1,49 @@ +# @contentstack/cli-asset-management + +Asset Management 2.0 API adapter for Contentstack CLI export and import. Used by the export and import plugins when Asset Management (AM 2.0) is enabled. To learn how to export and import content in Contentstack, refer to the [Migration guide](https://www.contentstack.com/docs/developers/cli/migration/). + +[![License](https://img.shields.io/npm/l/@contentstack/cli)](https://github.com/contentstack/cli/blob/main/LICENSE) + + +* [@contentstack/cli-asset-management](#contentstackcli-asset-management) +* [Overview](#overview) +* [Usage](#usage) +* [Exports](#exports) + + +# Overview + +This package provides: + +- **AssetManagementAdapter** – HTTP client for the Asset Management API (spaces, assets, folders, fields, asset types). +- **exportSpaceStructure** – Exports space metadata and full workspace structure (metadata, folders, assets, fields, asset types) for linked workspaces. +- **Types** – `AssetManagementExportOptions`, `LinkedWorkspace`, `IAssetManagementAdapter`, and related types for export/import integration. + +# Usage + +This package is consumed by the export and import plugins. When using the export CLI with the `--asset-management` flag (or when the host app enables AM 2.0), the export plugin calls `exportSpaceStructure` with linked workspaces and options: + +```ts +import { exportSpaceStructure } from '@contentstack/cli-asset-management'; + +await exportSpaceStructure({ + linkedWorkspaces, + exportDir, + branchName: 'main', + assetManagementUrl, + org_uid, + context, + progressManager, + progressProcessName, + updateStatus, + downloadAsset, // optional +}); +``` + +# Exports + +| Export | Description | +|--------|-------------| +| `exportSpaceStructure` | Async function to export space structure for given linked workspaces. | +| `AssetManagementAdapter` | Class to call the Asset Management API (getSpace, getWorkspaceFields, getWorkspaceAssets, etc.). | +| Types from `./types` | `AssetManagementExportOptions`, `ExportSpaceOptions`, `ChunkedJsonWriteOptions`, `LinkedWorkspace`, `SpaceResponse`, `FieldsResponse`, `AssetTypesResponse`, and related API types. | \ No newline at end of file diff --git a/packages/contentstack-asset-management/eslint.config.js b/packages/contentstack-asset-management/eslint.config.js new file mode 100644 index 000000000..1712e9ecd --- /dev/null +++ b/packages/contentstack-asset-management/eslint.config.js @@ -0,0 +1,60 @@ +import tseslint from 'typescript-eslint'; +import globals from 'globals'; +import oclif from 'eslint-config-oclif-typescript'; + +export default [ + ...tseslint.configs.recommended, + oclif, + { + languageOptions: { + parser: tseslint.parser, + parserOptions: { + project: './tsconfig.json', + }, + sourceType: 'module', + globals: { + ...globals.node, + }, + }, + plugins: { + '@typescript-eslint': tseslint.plugin, + }, + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { + args: 'none', + }, + ], + '@typescript-eslint/prefer-namespace-keyword': 'error', + '@typescript-eslint/quotes': [ + 'error', + 'single', + { + avoidEscape: true, + allowTemplateLiterals: true, + }, + ], + semi: 'off', + '@typescript-eslint/type-annotation-spacing': 'error', + '@typescript-eslint/no-redeclare': 'off', + eqeqeq: ['error', 'smart'], + 'id-match': 'error', + 'no-eval': 'error', + 'no-var': 'error', + quotes: 'off', + indent: 'off', + camelcase: 'off', + 'comma-dangle': 'off', + 'arrow-parens': 'off', + 'operator-linebreak': 'off', + 'object-curly-spacing': 'off', + 'node/no-missing-import': 'off', + 'padding-line-between-statements': 'off', + '@typescript-eslint/ban-ts-ignore': 'off', + 'unicorn/no-abusive-eslint-disable': 'off', + 'unicorn/consistent-function-scoping': 'off', + '@typescript-eslint/no-use-before-define': 'off', + }, + }, +]; diff --git a/packages/contentstack-asset-management/package.json b/packages/contentstack-asset-management/package.json index 9b0439ca7..750549ad1 100644 --- a/packages/contentstack-asset-management/package.json +++ b/packages/contentstack-asset-management/package.json @@ -1,6 +1,6 @@ { "name": "@contentstack/cli-asset-management", - "version": "1.0.0-beta.4", + "version": "1.0.0-beta.5", "description": "Contentstack Assets API adapter for export and import", "main": "lib/index.js", "types": "lib/index.d.ts", @@ -30,7 +30,8 @@ ], "license": "MIT", "dependencies": { - "@contentstack/cli-utilities": "~2.0.0-beta.9" + "@contentstack/cli-utilities": "~2.0.0-beta.10", + "lodash": "^4.18.1" }, "oclif": { "commands": "./lib/commands", @@ -42,11 +43,12 @@ }, "devDependencies": { "@types/chai": "^4.3.11", + "@types/lodash": "^4.17.0", "@types/mocha": "^10.0.6", "@types/node": "^20.17.50", "@types/sinon": "^17.0.4", "chai": "^4.4.1", - "eslint": "^8.57.1", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.68", "mocha": "^10.8.2", "nyc": "^15.1.0", @@ -54,6 +56,6 @@ "sinon": "^17.0.2", "source-map-support": "^0.5.21", "ts-node": "^10.9.2", - "typescript": "^5.8.3" + "typescript": "^5.9.3" } } diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 9dbc66186..ec288b72a 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -4,6 +4,8 @@ export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; export const FALLBACK_AM_API_CONCURRENCY = 5; /** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; +export const FALLBACK_AM_API_PAGE_SIZE = 100; +export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5; /** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ diff --git a/packages/contentstack-asset-management/src/export/asset-types.ts b/packages/contentstack-asset-management/src/export/asset-types.ts index 50487195e..7b171a25f 100644 --- a/packages/contentstack-asset-management/src/export/asset-types.ts +++ b/packages/contentstack-asset-management/src/export/asset-types.ts @@ -18,7 +18,7 @@ export default class ExportAssetTypes extends CSAssetsExportAdapter { log.debug('Starting shared asset types export process...', this.exportContext.context); - const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid); + const assetTypesData = await this.getWorkspaceAssetTypes(spaceUid, this.apiPageSize, this.apiFetchConcurrency); const items = getArrayFromResponse(assetTypesData, 'asset_types'); const dir = this.getAssetTypesDir(); if (items.length === 0) { diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 6cc1129a3..8fdad2039 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -1,20 +1,31 @@ import { resolve as pResolve } from 'node:path'; import { Readable } from 'node:stream'; import { mkdir, writeFile } from 'node:fs/promises'; -import { configHandler, log } from '@contentstack/cli-utilities'; +import chunk from 'lodash/chunk'; +import { configHandler, log, FsUtility } from '@contentstack/cli-utilities'; import type { CSAssetsAPIConfig, LinkedWorkspace } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsExportAdapter } from './base'; -import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; -import { runInBatches } from '../utils/concurrent-batch'; +import { writeStreamToFile } from '../utils/export-helpers'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry'; +import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter'; import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid']; + +type AssetRecord = { uid?: string; _uid?: string; url?: string; filename?: string; file_name?: string }; + export default class ExportAssets extends CSAssetsExportAdapter { constructor(apiConfig: CSAssetsAPIConfig, exportContext: ExportContext) { super(apiConfig, exportContext); } + private isDownloadable(asset: AssetRecord): boolean { + return Boolean(asset?.url && (asset?.uid ?? asset?._uid)); + } + async start(workspace: LinkedWorkspace, spaceDir: string): Promise { await this.init(); @@ -25,113 +36,142 @@ export default class ExportAssets extends CSAssetsExportAdapter { await mkdir(assetsDir, { recursive: true }); log.debug(`Assets directory ready: ${assetsDir}`, this.exportContext.context); - log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); - - const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid, workspace.uid), - this.getWorkspaceAssets(workspace.space_uid, workspace.uid), + // Stream asset metadata straight to chunked JSON as pages arrive — never hold the full set in + // memory. The writer is created lazily so an empty space writes an empty index instead of chunks. + let fsWriter: FsUtility | undefined; + let totalStreamed = 0; + let downloadableCount = 0; + const onPage = (items: unknown[]) => { + if (items.length === 0) return; + if (!fsWriter) fsWriter = this.createChunkedJsonWriter(assetsDir, 'assets.json', 'assets', ASSET_META_KEYS); + fsWriter.writeIntoFile(items as Record[], { mapKeyVal: true }); + totalStreamed += items.length; + for (const asset of items as AssetRecord[]) if (this.isDownloadable(asset)) downloadableCount += 1; + }; + + log.debug(`Fetching folders and streaming assets for space ${workspace.space_uid}`, this.exportContext.context); + const [folders] = await Promise.all([ + this.getWorkspaceFolders(workspace.space_uid, workspace.uid, this.apiPageSize, this.apiFetchConcurrency), + this.streamWorkspaceAssets(workspace.space_uid, workspace.uid, onPage, this.apiPageSize, this.apiFetchConcurrency), ]); - const assetItems = getAssetItems(assetsData); - const downloadableCount = assetItems.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))).length; + if (fsWriter) fsWriter.completeFile(true); + else await this.writeEmptyChunkedJson(assetsDir, 'assets.json'); + log.debug(`Wrote chunked assets metadata (${totalStreamed} item(s)) under ${assetsDir}`, this.exportContext.context); + // Per-space total: 1 folder write + 1 metadata write + N per-asset downloads. - // The shared module-level total is just a placeholder before this point; update - // it now so the multibar row shows real progress as downloads tick in. this.progressOrParent?.updateProcessTotal?.(this.processName, 2 + downloadableCount); await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); this.tick(true, `folders: ${workspace.space_uid}`, null); log.debug(`Wrote folders.json for space ${workspace.space_uid}`, this.exportContext.context); - log.debug( - assetItems.length === 0 - ? `No assets for space ${workspace.space_uid}, wrote empty assets.json` - : `Writing ${assetItems.length} assets metadata for space ${workspace.space_uid}`, - this.exportContext.context, - ); - await this.writeItemsToChunkedJson( - assetsDir, - 'assets.json', - 'assets', - ['uid', 'url', 'filename', 'file_name', 'parent_uid'], - assetItems, - ); - log.debug( - `Finished writing chunked assets metadata (${assetItems.length} item(s)) under ${assetsDir}`, - this.exportContext.context, - ); log.info( - assetItems.length === 0 + totalStreamed === 0 ? `Wrote empty asset metadata for space ${workspace.space_uid}` - : `Wrote ${assetItems.length} asset metadata record(s) for space ${workspace.space_uid}`, + : `Wrote ${totalStreamed} asset metadata record(s) for space ${workspace.space_uid}`, this.exportContext.context, ); - this.tick(true, `metadata: ${workspace.space_uid} (${assetItems.length})`, null); + this.tick(true, `metadata: ${workspace.space_uid} (${totalStreamed})`, null); log.debug(`Starting binary downloads for space ${workspace.space_uid}`, this.exportContext.context); - await this.downloadWorkspaceAssets(assetsData, assetsDir, workspace.space_uid); + await this.downloadWorkspaceAssets(assetsDir, workspace.space_uid, downloadableCount); } - private async downloadWorkspaceAssets(assetsData: unknown, assetsDir: string, spaceUid: string): Promise { - const items = getAssetItems(assetsData); - if (items.length === 0) { - log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context); - log.debug('No assets to download', this.exportContext.context); - return; - } - - this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING); - log.info(`Downloading asset files for space ${spaceUid} (${items.length} in metadata)`, this.exportContext.context); - log.debug(`Downloading ${items.length} asset file(s) for space ${spaceUid}...`, this.exportContext.context); + /** + * Download asset binaries by reading the just-written chunked `assets.json` back from disk + * (one chunk at a time), so we never re-materialize the whole asset list in memory. + */ + private async downloadWorkspaceAssets(assetsDir: string, spaceUid: string, expectedDownloads: number): Promise { const filesDir = pResolve(assetsDir, 'files'); await mkdir(filesDir, { recursive: true }); - log.debug(`Asset files directory ready: ${filesDir}`, this.exportContext.context); const securedAssets = this.exportContext.securedAssets ?? false; const authtoken = securedAssets ? configHandler.get('authtoken') : null; log.debug( - `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}`, + `Asset downloads: securedAssets=${securedAssets}, concurrency=${this.downloadAssetsBatchConcurrency}, expected=${expectedDownloads}`, this.exportContext.context, ); + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].DOWNLOADING); + let downloadOk = 0; let downloadFail = 0; - const validItems = items.filter((asset) => Boolean(asset.url && (asset.uid ?? asset._uid))); - const skipped = items.length - validItems.length; - if (skipped > 0) { - log.debug( - `Skipping ${skipped} asset row(s) without url or uid (${validItems.length} file download(s) scheduled)`, + await forEachChunkedJsonStore( + assetsDir, + 'assets.json', + { + context: this.exportContext.context, + chunkReadLogLabel: 'assets', + onOpenError: (err) => log.debug(`Could not open assets.json for download: ${err}`, this.exportContext.context), + onEmptyIndexer: () => log.info(`No asset files to download for space ${spaceUid}`, this.exportContext.context), + }, + async (records) => { + const valid = records.filter((asset) => this.isDownloadable(asset)); + if (valid.length === 0) return; + const apiBatches = chunk(valid, this.downloadAssetsBatchConcurrency); + const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => { + const asset = apiBatches[batchIndex][index] as AssetRecord; + const uid = (asset.uid ?? asset._uid) as string; + const url = asset.url as string; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + if (!url || !uid) return; + try { + const separator = url.includes('?') ? '&' : '?'; + const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; + // Binary GET is idempotent — retry transient failures with backoff. + const response = await withRetry( + async () => { + let resp: Response; + try { + resp = await fetch(downloadUrl); + } catch (e) { + throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`); + } + if (!resp.ok) { + if (isRetryableStatus(resp.status)) { + throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after'))); + } + throw new Error(`HTTP ${resp.status}`); + } + return resp; + }, + { context: this.exportContext.context, label: `download ${filename}` }, + ); + const body = response.body; + if (!body) throw new Error('No response body'); + const nodeStream = Readable.fromWeb(body as Parameters[0]); + const assetFolderPath = pResolve(filesDir, uid); + await mkdir(assetFolderPath, { recursive: true }); + const filePath = pResolve(assetFolderPath, filename); + await writeStreamToFile(nodeStream, filePath); + downloadOk += 1; + // Per-asset tick so the per-space progress bar moves in real time. + this.tick(true, `asset: ${filename}`, null); + log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context); + } catch (e) { + downloadFail += 1; + const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; + this.tick(false, `asset: ${filename}`, err); + log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); + } + }; + + await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler); + }, + ); + + // Completeness check: a chunk that fails to read back is skipped (logged at debug) by + // forEachChunkedJsonStore, which would silently drop those downloads. Reconcile attempts + // (ok + failed) against what streaming counted as downloadable. + const attempted = downloadOk + downloadFail; + if (attempted < expectedDownloads) { + log.warn( + `Asset downloads for space ${spaceUid} incomplete: expected ${expectedDownloads}, attempted ${attempted}` + + ` — ${expectedDownloads - attempted} asset(s) were never read back for download.`, this.exportContext.context, ); } - await runInBatches(validItems, this.downloadAssetsBatchConcurrency, async (asset) => { - const uid = asset.uid ?? asset._uid; - const url = asset.url; - const filename = asset.filename ?? asset.file_name ?? 'asset'; - if (!url || !uid) return; - try { - const separator = url.includes('?') ? '&' : '?'; - const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; - const response = await fetch(downloadUrl); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const body = response.body; - if (!body) throw new Error('No response body'); - const nodeStream = Readable.fromWeb(body as Parameters[0]); - const assetFolderPath = pResolve(filesDir, uid); - await mkdir(assetFolderPath, { recursive: true }); - const filePath = pResolve(assetFolderPath, filename); - await writeStreamToFile(nodeStream, filePath); - downloadOk += 1; - // Per-asset tick so the per-space progress bar moves in real time. - this.tick(true, `asset: ${filename}`, null); - log.debug(`Downloaded asset ${uid} → ${filePath}`, this.exportContext.context); - } catch (e) { - downloadFail += 1; - const err = (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_DOWNLOADS].FAILED; - this.tick(false, `asset: ${filename}`, err); - log.debug(`Failed to download asset ${uid}: ${e}`, this.exportContext.context); - } - }); log.info( downloadFail === 0 diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index b9721685c..13e4016e1 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,7 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { CSAssetsAPIConfig } from '../types/cs-assets-api'; import type { ExportContext } from '../types/export-types'; import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter'; -import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; +import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY, FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; export type { ExportContext }; @@ -82,6 +82,14 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { return this.exportContext.downloadAssetsConcurrency ?? this.apiConcurrency; } + protected get apiPageSize(): number { + return this.exportContext.pageSize ?? FALLBACK_AM_API_PAGE_SIZE; + } + + protected get apiFetchConcurrency(): number { + return this.exportContext.fetchConcurrency ?? FALLBACK_AM_API_FETCH_CONCURRENCY; + } + protected getAssetTypesDir(): string { return pResolve(this.exportContext.spacesRootPath, 'asset_types'); } @@ -90,6 +98,25 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { return pResolve(this.exportContext.spacesRootPath, 'fields'); } + /** Build a chunked-JSON writer for incremental (streaming) writes. Caller must `completeFile(true)`. */ + protected createChunkedJsonWriter(dir: string, indexFileName: string, moduleName: string, metaPickKeys: string[]): FsUtility { + const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; + return new FsUtility({ + basePath: dir, + indexFileName, + chunkFileSize: chunkMb, + moduleName, + fileExt: 'json', + metaPickKeys, + keepMetadata: true, + }); + } + + /** Write an empty index file (matches FsUtility's layout for a zero-record store). */ + protected async writeEmptyChunkedJson(dir: string, indexFileName: string): Promise { + await writeFile(pResolve(dir, indexFileName), '{}'); + } + protected async writeItemsToChunkedJson( dir: string, indexFileName: string, @@ -98,19 +125,10 @@ export class CSAssetsExportAdapter extends CSAssetsAdapter { items: unknown[], ): Promise { if (items.length === 0) { - await writeFile(pResolve(dir, indexFileName), '{}'); + await this.writeEmptyChunkedJson(dir, indexFileName); return; } - const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; - const fs = new FsUtility({ - basePath: dir, - indexFileName, - chunkFileSize: chunkMb, - moduleName, - fileExt: 'json', - metaPickKeys, - keepMetadata: true, - }); + const fs = this.createChunkedJsonWriter(dir, indexFileName, moduleName, metaPickKeys); fs.writeIntoFile(items as Record[], { mapKeyVal: true }); fs.completeFile(true); } diff --git a/packages/contentstack-asset-management/src/export/fields.ts b/packages/contentstack-asset-management/src/export/fields.ts index c1ca623f8..a4cfe8460 100644 --- a/packages/contentstack-asset-management/src/export/fields.ts +++ b/packages/contentstack-asset-management/src/export/fields.ts @@ -18,7 +18,7 @@ export default class ExportFields extends CSAssetsExportAdapter { log.debug('Starting shared fields export process...', this.exportContext.context); - const fieldsData = await this.getWorkspaceFields(spaceUid); + const fieldsData = await this.getWorkspaceFields(spaceUid, this.apiPageSize, this.apiFetchConcurrency); const items = getArrayFromResponse(fieldsData, 'fields'); const dir = this.getFieldsDir(); if (items.length === 0) { diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index 3a3459c3f..6d3b0ab13 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -79,6 +79,8 @@ export class ExportSpaces { chunkFileSizeMb, apiConcurrency: this.options.apiConcurrency, downloadAssetsConcurrency: this.options.downloadAssetsConcurrency, + pageSize: this.options.pageSize, + fetchConcurrency: this.options.fetchConcurrency, }; const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); diff --git a/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts b/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts index 5ea914e95..dca54810b 100644 --- a/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts +++ b/packages/contentstack-asset-management/src/import-setup/import-setup-asset-mappers.ts @@ -4,7 +4,7 @@ import { join, resolve } from 'node:path'; import { formatError, log } from '@contentstack/cli-utilities'; -import { IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE, IMPORT_ASSETS_MAPPER_FILES, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; import type { CSAssetsAPIConfig, ImportContext } from '../types/cs-assets-api'; import type { AssetMapperImportSetupResult, RunAssetMapperImportSetupParams } from '../types/import-setup-asset-mapper'; import ImportAssets from '../import/assets'; @@ -25,7 +25,7 @@ export default class ImportSetupAssetMappers extends AssetManagementImportSetupA private async fetchExistingSpaceUidsInOrg(apiConfig: CSAssetsAPIConfig): Promise> { const adapter = new CSAssetsAdapter(apiConfig); await adapter.init(); - const { spaces } = await adapter.listSpaces(); + const { spaces } = await adapter.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY); const uids = new Set(); for (const s of spaces) { if (s.uid) { diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts index 457c1f36d..faec9d13d 100644 --- a/packages/contentstack-asset-management/src/import/spaces.ts +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -10,7 +10,7 @@ import type { ImportSpacesOptions, SpaceMapping, } from '../types/cs-assets-api'; -import { CS_ASSETS_MAIN_PROCESS_NAME, PROCESS_NAMES, getSpaceProcessName } from '../constants/index'; +import { CS_ASSETS_MAIN_PROCESS_NAME, FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY, PROCESS_NAMES, getSpaceProcessName } from '../constants/index'; import { CSAssetsAdapter } from '../utils/cs-assets-api-adapter'; import ImportAssetTypes from './asset-types'; import ImportFields from './fields'; @@ -112,7 +112,7 @@ export class ImportSpaces { try { const adapterForList = new CSAssetsAdapter(apiConfig); await adapterForList.init(); - const { spaces } = await adapterForList.listSpaces(); + const { spaces } = await adapterForList.listSpaces(FALLBACK_AM_API_PAGE_SIZE, FALLBACK_AM_API_FETCH_CONCURRENCY); for (const s of spaces) { if (s.uid) existingSpaceUids.add(s.uid); } diff --git a/packages/contentstack-asset-management/src/import/workspaces.ts b/packages/contentstack-asset-management/src/import/workspaces.ts index 381771d30..369afe15e 100644 --- a/packages/contentstack-asset-management/src/import/workspaces.ts +++ b/packages/contentstack-asset-management/src/import/workspaces.ts @@ -66,20 +66,6 @@ export default class ImportWorkspace extends CSAssetsImportAdapter { assetsImporter.setProcessName(spaceProcessName); } - // Map source default space → existing target default space (cross-org migration). - // The caller supplies the uid of the pre-existing target default space; we upload - // source assets into it instead of creating a new space. - if (isDefault && targetDefaultSpaceUid) { - const newSpaceUid = targetDefaultSpaceUid; - const resolvedWorkspaceUid = targetDefaultWorkspaceUid ?? workspaceUid; - log.info( - `Source default space "${oldSpaceUid}" mapped to existing target default space "${newSpaceUid}".`, - this.importContext.context, - ); - const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); - return { oldSpaceUid, newSpaceUid, workspaceUid: resolvedWorkspaceUid, isDefault: true, uidMap, urlMap }; - } - // Reuse: target org already has a space with the same uid as the export directory. if (existingSpaceUids.has(oldSpaceUid)) { log.info( @@ -100,6 +86,20 @@ export default class ImportWorkspace extends CSAssetsImportAdapter { urlMap, }; } + + // Map source default space → existing target default space (cross-org migration). + // The caller supplies the uid of the pre-existing target default space; we upload + // source assets into it instead of creating a new space. + if (isDefault && targetDefaultSpaceUid) { + const newSpaceUid = targetDefaultSpaceUid; + const resolvedWorkspaceUid = targetDefaultWorkspaceUid ?? workspaceUid; + log.info( + `Source default space "${oldSpaceUid}" mapped to existing target default space "${newSpaceUid}".`, + this.importContext.context, + ); + const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); + return { oldSpaceUid, newSpaceUid, workspaceUid: resolvedWorkspaceUid, isDefault: true, uidMap, urlMap }; + } // Create new space with exact exported title log.debug(`Creating space "${exportedTitle}" (old uid: ${oldSpaceUid})`, this.importContext.context); diff --git a/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts index 9e50ae94b..eda828159 100644 --- a/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts +++ b/packages/contentstack-asset-management/src/query-export/cs-assets-query-exporter.ts @@ -8,8 +8,10 @@ import type { ExportContext } from '../types/export-types'; import ExportAssetTypes from '../export/asset-types'; import ExportFields from '../export/fields'; import { CSAssetsExportAdapter } from '../export/base'; +import chunk from 'lodash/chunk'; import { getAssetItems, writeStreamToFile } from '../utils/export-helpers'; -import { runInBatches } from '../utils/concurrent-batch'; +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../utils/retry'; +import type { CustomPromiseHandler } from '../utils/cs-assets-api-adapter'; const DEFAULT_ASSET_BATCH_SIZE = 100; const SEARCH_PAGE_LIMIT = 100; @@ -223,15 +225,34 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { const securedAssets = this.exportContext.securedAssets ?? false; const authtoken = securedAssets ? configHandler.get('authtoken') : null; - await runInBatches(downloadable, this.downloadAssetsBatchConcurrency, async (asset) => { + const apiBatches = chunk(downloadable, this.downloadAssetsBatchConcurrency); + const promisifyHandler: CustomPromiseHandler = async ({ index, batchIndex }) => { + const asset = apiBatches[batchIndex][index]; const uid = String(asset.uid ?? asset._uid); const url = String(asset.url); const filename = String(asset.filename ?? asset.file_name ?? 'asset'); try { const separator = url.includes('?') ? '&' : '?'; const downloadUrl = securedAssets && authtoken ? `${url}${separator}authtoken=${authtoken}` : url; - const response = await fetch(downloadUrl); - if (!response.ok) throw new Error(`HTTP ${response.status}`); + // Binary GET is idempotent — retry transient failures with backoff. + const response = await withRetry( + async () => { + let resp: Response; + try { + resp = await fetch(downloadUrl); + } catch (e) { + throw new RetryableHttpError(`download network error: ${(e as Error)?.message ?? String(e)}`); + } + if (!resp.ok) { + if (isRetryableStatus(resp.status)) { + throw new RetryableHttpError(`HTTP ${resp.status}`, resp.status, parseRetryAfterMs(resp.headers.get('retry-after'))); + } + throw new Error(`HTTP ${resp.status}`); + } + return resp; + }, + { context: this.exportContext.context, label: `download ${filename}` }, + ); const body = response.body; if (!body) throw new Error('No response body'); const nodeStream = Readable.fromWeb(body as Parameters[0]); @@ -241,6 +262,8 @@ class QueryExportWorkspaceAdapter extends CSAssetsExportAdapter { } catch (e) { log.debug(`Failed to download asset ${uid} in space ${spaceUid}: ${e}`, this.exportContext.context); } - }); + }; + + await this.makeConcurrentCall({ apiBatches, module: 'asset downloads' }, promisifyHandler); } } diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 96dbac1bd..56e45bfff 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -113,6 +113,10 @@ export type CSAssetsAPIConfig = { headers?: Record; /** Optional context for logging (e.g. exportConfig.context) */ context?: Record; + /** Max retry attempts for transient read failures (network/429/5xx). Default 3. */ + retries?: number; + /** Base backoff (ms) for retries; actual delay grows exponentially with jitter. Default 500. */ + retryBaseDelayMs?: number; }; // --------------------------------------------------------------------------- @@ -167,12 +171,19 @@ export type SearchAssetsResponse = { export interface ICSAssetsAdapter { init(): Promise; - listSpaces(): Promise; + listSpaces(pageSize?: number, fetchConcurrency?: number): Promise; getSpace(spaceUid: string): Promise; - getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; - getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; - getWorkspaceAssetTypes(spaceUid: string): Promise; + getWorkspaceFields(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; + streamWorkspaceAssets( + spaceUid: string, + workspaceUid: string | undefined, + onPage: (items: unknown[]) => void | Promise, + pageSize?: number, + fetchConcurrency?: number, + ): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize?: number, fetchConcurrency?: number): Promise; + getWorkspaceAssetTypes(spaceUid: string, pageSize?: number, fetchConcurrency?: number): Promise; searchAssets(params: SearchAssetsParams): Promise; bulkDeleteAssets( spaceUid: string, @@ -233,6 +244,8 @@ export type AssetManagementExportOptions = { * Max parallel asset file downloads per workspace. */ downloadAssetsConcurrency?: number; + pageSize?: number; + fetchConcurrency?: number; }; // --------------------------------------------------------------------------- diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index 865302a60..3e21682b4 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -5,6 +5,8 @@ export type ExportContext = { chunkFileSizeMb?: number; apiConcurrency?: number; downloadAssetsConcurrency?: number; + pageSize?: number; + fetchConcurrency?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/concurrent-batch.ts b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts index dcd916c4b..727c6f0ff 100644 --- a/packages/contentstack-asset-management/src/utils/concurrent-batch.ts +++ b/packages/contentstack-asset-management/src/utils/concurrent-batch.ts @@ -1,3 +1,12 @@ +/** + * Fault-tolerant batched concurrency for the import side (uploads, folder/asset-type/ + * field creation). `runInBatches` runs work in batches of `concurrency`, settling each + * batch (`Promise.allSettled`) before the next so one failure doesn't abort the batch. + * + * NOTE: the export side (pagination + downloads) uses the legacy-style `makeConcurrentCall` + * on `CSAssetsAdapter` instead; do not route export work through here. + */ + /** * Split an array into chunks of at most `size` elements. */ diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index f76fa4106..e10761ac2 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -1,7 +1,11 @@ import { readFileSync } from 'node:fs'; import { basename } from 'node:path'; +import chunk from 'lodash/chunk'; import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities'; +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from './retry'; +import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '../constants/index'; + import type { CSAssetsAPIConfig, AssetTypesResponse, @@ -51,6 +55,42 @@ export const DEFAULT_SEARCH_ASSET_FIELDS = [ '_asset_scan_status', ] as const; +/** + * Concurrency model ported from the legacy `contentstack-export` package + * (`src/export/modules/base-class.ts`). `makeConcurrentCall` runs work in + * batches of `concurrencyLimit`, settling each batch before the next and + * throttling between batches. Transport differs from legacy: `makeAPICall` + * dispatches to this adapter's HttpClient (`getSpaceLevel`) instead of the SDK. + */ +export type ApiModuleType = 'paginated-collection'; + +export type ApiOptions = { + uid?: string; + url?: string; + module: ApiModuleType; + queryParam?: Record; + resolve: (value: any) => void; + reject: (error: any) => void; + additionalInfo?: Record; +}; + +export type EnvType = { + module: string; + /** Pre-chunked work: each inner array runs in parallel, the outer array runs sequentially. */ + apiBatches: any[][]; + apiParams?: ApiOptions; +}; + +export type CustomPromiseHandlerInput = { + index: number; + batchIndex: number; + element?: any; + apiParams?: ApiOptions; + isLastRequest: boolean; +}; + +export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise; + export class CSAssetsAdapter implements ICSAssetsAdapter { private readonly config: CSAssetsAPIConfig; private readonly apiClient: HttpClient; @@ -144,26 +184,38 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { log.debug(`GET ${fullPath}`, this.config.context); try { - const response = await this.apiClient.get(fullPath); - if (response.status < 200 || response.status >= 300) { - const bodySnippet = this.formatResponseBodyForError(response.data); - throw this.normalizeAmGetFailure({ - path, - fullPath, - status: response.status, - bodySnippet: bodySnippet || undefined, - }); - } - return response.data as T; + // GETs are idempotent, so retry transient failures (network / 429 / 5xx) with backoff. + return await withRetry( + async () => { + let response: Awaited>; + try { + response = await this.apiClient.get(fullPath); + } catch (netErr) { + // Transport-level rejection (connection reset, timeout, DNS) — transient. + throw new RetryableHttpError(`network error: ${(netErr as Error)?.message ?? String(netErr)}`); + } + if (response.status < 200 || response.status >= 300) { + if (isRetryableStatus(response.status)) { + const retryAfter = parseRetryAfterMs((response as { headers?: Record })?.headers?.['retry-after']); + throw new RetryableHttpError(`GET ${fullPath} → ${response.status}`, response.status, retryAfter); + } + // Terminal (e.g. 4xx): normalize and propagate without retrying. + const bodySnippet = this.formatResponseBodyForError(response.data); + throw this.normalizeAmGetFailure({ path, fullPath, status: response.status, bodySnippet: bodySnippet || undefined }); + } + return response.data as T; + }, + { retries: this.config.retries, baseDelayMs: this.config.retryBaseDelayMs, context: this.config.context, label: `GET ${path}` }, + ); } catch (error) { + if (error instanceof RetryableHttpError) { + // Retries exhausted on a transient failure — surface a normalized error to the caller. + throw this.normalizeAmGetFailure({ path, fullPath, status: error.status, cause: error }); + } if (error instanceof Error && error.message.includes('CS Assets API GET failed')) { throw error; } - throw this.normalizeAmGetFailure({ - path, - fullPath, - cause: error, - }); + throw this.normalizeAmGetFailure({ path, fullPath, cause: error }); } } @@ -189,11 +241,17 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } } - async listSpaces(): Promise { + async listSpaces(pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { log.debug('Fetching all spaces in org', this.config.context); - const result = await this.getSpaceLevel('', '/api/spaces', {}); - log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); - return result; + const items = await this.fetchAllPages( + '', + '/api/spaces', + 'spaces', + pageSize, + fetchConcurrency, + ); + log.debug(`Fetched ${items.length} space(s)`, this.config.context); + return { spaces: items as Space[], count: items.length }; } async getSpace(spaceUid: string): Promise { @@ -207,11 +265,15 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } - async getWorkspaceFields(spaceUid: string): Promise { + async getWorkspaceFields( + spaceUid: string, + pageSize = FALLBACK_AM_API_PAGE_SIZE, + fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, + ): Promise { log.debug(`Fetching fields for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, '/api/fields', {}); - log.debug(`Fetched fields (count: ${result?.count ?? '?'})`, this.config.context); - return result; + const items = await this.fetchAllPages(spaceUid, '/api/fields', 'fields', pageSize, fetchConcurrency, {}); + log.debug(`Fetched fields (count: ${items.length})`, this.config.context); + return { fields: items, count: items.length } as FieldsResponse; } /** @@ -230,31 +292,238 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { return result; } - async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { - return this.getWorkspaceCollection( + + /** + * Core pagination: read the total `count` from page 0, then drive the remaining pages through + * {@link makeConcurrentCall}. Every page (including page 0) is handed to `onPage` — writes are + * serialized through a promise chain so a streaming sink (e.g. FsUtility) is never called + * reentrantly while pages fetch concurrently. Returns the number of items seen. + * + * Peak memory is bounded by the sink: the array wrapper holds everything, but a disk-writing + * sink keeps only the in-flight pages (~concurrency × pageSize). + */ + private async paginate( + spaceUid: string, + path: string, + itemsKey: string, + pageSize: number, + concurrency: number, + baseParams: Record, + onPage: (items: unknown[]) => void | Promise, + ): Promise { + const first = await this.getSpaceLevel>(spaceUid, path, { + ...baseParams, limit: String(pageSize), skip: '0', + }); + + const total: number = Number(first?.count ?? 0); + const firstItems: unknown[] = Array.isArray(first?.[itemsKey]) ? (first[itemsKey] as unknown[]) : []; + + let collected = 0; + let writeFailures = 0; + let writeChain: Promise = Promise.resolve(); + const enqueue = (items: unknown[]) => { + collected += items.length; + // Each link catches its own error so a single failed sink write doesn't skip the queued ones. + writeChain = writeChain.then(async () => { + try { + await onPage(items); + } catch (e) { + writeFailures += 1; + log.warn(`Failed to persist a page of ${itemsKey} (${path}): ${(e as Error)?.message ?? e}`, this.config.context); + } + }); + }; + + enqueue(firstItems); + + if (firstItems.length < total) { + // Remaining skip offsets (page 0 already fetched), pre-chunked into batches of `concurrency`. + const skips: string[] = Array.from( + { length: Math.ceil(total / pageSize) - 1 }, + (_, i) => String((i + 1) * pageSize), + ); + const apiBatches = chunk(skips, concurrency); + + let failedPages = 0; + const onSuccess = ({ response }: any) => { + const items = Array.isArray(response?.[itemsKey]) ? (response[itemsKey] as unknown[]) : []; + enqueue(items); + }; + const onReject = ({ error }: any) => { + // A failed page is skipped (Promise.allSettled); surface it loudly rather than silently dropping data. + failedPages += 1; + log.warn(`Failed to fetch a page of ${itemsKey} (${path}): ${error?.message ?? error}`, this.config.context); + }; + + await this.makeConcurrentCall({ + module: itemsKey, + apiBatches, + apiParams: { + module: 'paginated-collection', + resolve: onSuccess, + reject: onReject, + queryParam: { ...baseParams, limit: String(pageSize) }, + additionalInfo: { spaceUid, path, itemsKey }, + }, + }); + + // Completeness check: the export "succeeding" with silently-missing pages is the worst failure + // mode for a backup/migration, so reconcile what we saw against the server's reported total. + if (collected !== total) { + log.warn( + `Incomplete pagination for ${itemsKey} (${path}): expected ${total}, collected ${collected}` + + (failedPages > 0 ? ` — ${failedPages} page request(s) failed.` : '.'), + this.config.context, + ); + } + } + + await writeChain; // flush any queued sink writes before returning + if (writeFailures > 0) { + log.warn( + `${writeFailures} page(s) of ${itemsKey} (${path}) failed to persist — output may be incomplete.`, + this.config.context, + ); + } + return collected; + } + + /** + * Fetch all pages of a paginated collection into an in-memory array. Use for small collections + * (spaces/folders/fields/asset-types); for potentially large asset sets prefer + * {@link streamWorkspaceAssets}, which streams to a sink instead of accumulating. + */ + private async fetchAllPages( + spaceUid: string, + path: string, + itemsKey: string, + pageSize: number, + concurrency: number, + baseParams: Record = {}, + ): Promise { + const out: unknown[] = []; + await this.paginate(spaceUid, path, itemsKey, pageSize, concurrency, baseParams, (items) => { + out.push(...items); + }); + return out; + } + + /** + * Stream a workspace's assets page-by-page to `onPage` (e.g. an incremental chunked-JSON writer) + * instead of buffering the whole set. Returns the number of asset records streamed. + */ + async streamWorkspaceAssets( + spaceUid: string, + workspaceUid: string | undefined, + onPage: (items: unknown[]) => void | Promise, + pageSize = FALLBACK_AM_API_PAGE_SIZE, + fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, + ): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + return this.paginate( + spaceUid, + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + 'assets', + pageSize, + fetchConcurrency, + baseParams, + onPage, + ); + } + + /** + * Run pre-batched API work with bounded concurrency: each inner array of `apiBatches` + * runs in parallel (`Promise.allSettled`), and batches run sequentially. Either invokes a + * `promisifyHandler` per element, or — for paginated GETs — injects each element as `skip` + * and dispatches through {@link makeAPICall}. Adapted from legacy export's `makeConcurrentCall`. + * + * Callers pre-chunk the work (`chunk(items, concurrency)`), so this never derives batches itself. + */ + async makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler): Promise { + const { module, apiBatches, apiParams } = env; + if (!apiBatches?.length) return; + + for (let batchIndex = 0; batchIndex < apiBatches.length; batchIndex++) { + const currentBatch = apiBatches[batchIndex]; + const allPromise: Array> = []; + + for (let index = 0; index < currentBatch.length; index++) { + const element = currentBatch[index]; + const isLastRequest = batchIndex === apiBatches.length - 1 && index === currentBatch.length - 1; + + if (promisifyHandler) { + allPromise.push(promisifyHandler({ apiParams, element, isLastRequest, index, batchIndex })); + } else if (apiParams?.queryParam) { + // Mutated in place per iteration; makeAPICall snapshots it synchronously (see below). + apiParams.queryParam.skip = element; + allPromise.push(this.makeAPICall(apiParams, isLastRequest)); + } + } + + await Promise.allSettled(allPromise); + log.debug(`Batch ${batchIndex + 1}/${apiBatches.length} of ${module} complete`, this.config.context); + } + } + + /** + * Dispatch a single API call for {@link makeConcurrentCall}. Transport adapted from + * legacy's SDK calls to this adapter's HttpClient. `queryParam` is snapshotted + * synchronously (the caller mutates `skip` in place between iterations). + */ + makeAPICall( + { module: moduleName, reject, resolve, additionalInfo, queryParam = {} }: ApiOptions, + isLastRequest = false, + ): Promise { + switch (moduleName) { + case 'paginated-collection': { + const { spaceUid = '', path = '' } = (additionalInfo ?? {}) as { spaceUid?: string; path?: string }; + const params = { ...queryParam }; + return this.getSpaceLevel>(spaceUid, path, params) + .then((response: any) => resolve({ response, isLastRequest, additionalInfo })) + .catch((error: Error) => reject({ error, isLastRequest, additionalInfo })); + } + default: + return Promise.resolve(); + } + } + + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', - workspaceUid ? { workspace: workspaceUid } : {}, + pageSize, + fetchConcurrency, + baseParams, ); + return { assets: items, count: items.length }; } - async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { - return this.getWorkspaceCollection( + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string, pageSize = FALLBACK_AM_API_PAGE_SIZE, fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY): Promise { + const baseParams: Record = workspaceUid ? { workspace: workspaceUid } : {}; + const items = await this.fetchAllPages( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', - workspaceUid ? { workspace: workspaceUid } : {}, + pageSize, + fetchConcurrency, + baseParams, ); + return { folders: items, count: items.length }; } - async getWorkspaceAssetTypes(spaceUid: string): Promise { + async getWorkspaceAssetTypes( + spaceUid: string, + pageSize = FALLBACK_AM_API_PAGE_SIZE, + fetchConcurrency = FALLBACK_AM_API_FETCH_CONCURRENCY, + ): Promise { log.debug(`Fetching asset types for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, '/api/asset_types', { + const items = await this.fetchAllPages(spaceUid, '/api/asset_types', 'asset_types', pageSize, fetchConcurrency, { include_fields: 'true', }); - log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); - return result; + log.debug(`Fetched asset types (count: ${items.length})`, this.config.context); + return { asset_types: items, count: items.length } as AssetTypesResponse; } /** @@ -284,7 +553,8 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { `Searching assets (skip=${skip}, limit=${limit}, uids=${assetUIDs.length}, spaces=${spaces.length})`, this.config.context, ); - return this.postJson('/api/search', body); + // Search is a read — safe to retry transient failures. + return this.postJson('/api/search', body, {}, { retry: true }); } // --------------------------------------------------------------------------- @@ -309,18 +579,32 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { }; } - private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + /** + * POST a JSON body. Pass `{ retry: true }` ONLY for idempotent reads (e.g. /api/search) — never + * for writes (create/bulk), which could double-apply on retry. + */ + private async postJson( + path: string, + body: unknown, + extraHeaders: Record = {}, + opts: { retry?: boolean } = {}, + ): Promise { const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); log.debug(`POST ${path}`, this.config.context); - try { - const response = await fetch(`${baseUrl}${path}`, { - method: 'POST', - headers, - body: JSON.stringify(body), - }); + const doPost = async (): Promise => { + let response: Response; + try { + response = await fetch(`${baseUrl}${path}`, { method: 'POST', headers, body: JSON.stringify(body) }); + } catch (netErr) { + if (opts.retry) throw new RetryableHttpError(`POST ${path} network error: ${(netErr as Error)?.message ?? String(netErr)}`); + throw netErr; + } if (!response.ok) { + if (opts.retry && isRetryableStatus(response.status)) { + throw new RetryableHttpError(`POST ${path} → ${response.status}`, response.status, parseRetryAfterMs(response.headers.get('retry-after'))); + } const text = await response.text().catch(() => ''); const bodySnippet = this.formatResponseBodyForError(text); throw new Error( @@ -330,7 +614,21 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { ); } return response.json() as Promise; + }; + + try { + return opts.retry + ? await withRetry(doPost, { + retries: this.config.retries, + baseDelayMs: this.config.retryBaseDelayMs, + context: this.config.context, + label: `POST ${path}`, + }) + : await doPost(); } catch (error) { + if (error instanceof RetryableHttpError) { + throw new Error(`CS Assets API POST failed: path ${path} (status ${error.status ?? 'network'}) - ${error.message}`); + } if (error instanceof Error && error.message.includes('CS Assets API POST failed')) { throw error; } diff --git a/packages/contentstack-asset-management/src/utils/index.ts b/packages/contentstack-asset-management/src/utils/index.ts index dca27cade..b78e6bf0a 100644 --- a/packages/contentstack-asset-management/src/utils/index.ts +++ b/packages/contentstack-asset-management/src/utils/index.ts @@ -8,5 +8,6 @@ export { writeStreamToFile, } from './export-helpers'; export { chunkArray, runInBatches } from './concurrent-batch'; +export { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from './retry'; export { detectAssetManagementExportFromContentDir } from './detect-asset-management-export'; export type { AssetManagementExportFlags } from '../types/asset-management-export-flags'; diff --git a/packages/contentstack-asset-management/src/utils/retry.ts b/packages/contentstack-asset-management/src/utils/retry.ts new file mode 100644 index 000000000..1aed4cde9 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/retry.ts @@ -0,0 +1,87 @@ +import { log } from '@contentstack/cli-utilities'; + +export const DEFAULT_RETRIES = 3; +export const DEFAULT_RETRY_BASE_DELAY_MS = 500; +/** Hard ceiling on any single backoff — caps both exponential growth and a server-supplied Retry-After. */ +export const MAX_RETRY_BACKOFF_MS = 30_000; + +export type RetryOptions = { + /** Max retry attempts after the initial try (default 3). */ + retries?: number; + /** Base backoff in ms; actual delay is baseDelayMs * 2^attempt + jitter (default 500). */ + baseDelayMs?: number; + context?: Record; + /** Short label for retry log lines (e.g. "GET /api/fields"). */ + label?: string; +}; + +/** + * Error that marks an operation as worth retrying (transient network failure, 429, or 5xx). + * Anything that is NOT a RetryableHttpError is treated as terminal by {@link withRetry}. + */ +export class RetryableHttpError extends Error { + readonly status?: number; + readonly retryAfterMs?: number; + + constructor(message: string, status?: number, retryAfterMs?: number) { + super(message); + this.name = 'RetryableHttpError'; + this.status = status; + this.retryAfterMs = retryAfterMs; + } +} + +/** Transient HTTP statuses worth retrying. */ +export function isRetryableStatus(status: number): boolean { + return status === 429 || status >= 500; +} + +/** Parse a Retry-After header (delta-seconds or HTTP date) into milliseconds, or undefined. */ +export function parseRetryAfterMs(headerValue: string | null | undefined): number | undefined { + if (!headerValue) return undefined; + const seconds = Number(headerValue); + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000); + const dateMs = Date.parse(headerValue); + if (!Number.isNaN(dateMs)) return Math.max(0, dateMs - Date.now()); + return undefined; +} + +function sleep(ms: number): Promise { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms)); +} + +/** + * Run `fn`, retrying only when it throws a {@link RetryableHttpError}. Uses exponential backoff + * (`baseDelayMs * 2^attempt`) plus jitter, or the error's `retryAfterMs` when present. Terminal + * errors (anything that isn't a RetryableHttpError) propagate immediately, as does the last + * RetryableHttpError once attempts are exhausted. + * + * Wrap sites are responsible for classifying: throw RetryableHttpError for network errors / 429 / + * 5xx, and throw a plain error for non-retryable failures (e.g. 4xx). Only idempotent reads should + * be wrapped — never non-idempotent writes (uploads/creates), which could double-apply. + */ +export async function withRetry(fn: () => Promise, opts: RetryOptions = {}): Promise { + const retries = opts.retries ?? DEFAULT_RETRIES; + const baseDelayMs = opts.baseDelayMs ?? DEFAULT_RETRY_BASE_DELAY_MS; + let attempt = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + try { + return await fn(); + } catch (error) { + if (!(error instanceof RetryableHttpError) || attempt >= retries) throw error; + const jitter = Math.floor(Math.random() * baseDelayMs); + const backoff = baseDelayMs * 2 ** attempt + jitter; + // Clamp so a hostile/broken server's Retry-After (or runaway exponential) can't stall the export. + const delay = Math.min(error.retryAfterMs ?? backoff, MAX_RETRY_BACKOFF_MS); + attempt += 1; + log.debug( + `Retry ${attempt}/${retries} in ${delay}ms${opts.label ? ` (${opts.label})` : ''}: ${error.message}`, + opts.context, + ); + await sleep(delay); + } + } +} diff --git a/packages/contentstack-asset-management/test/unit/export/assets.test.ts b/packages/contentstack-asset-management/test/unit/export/assets.test.ts index f6a4bc61e..ae930da8b 100644 --- a/packages/contentstack-asset-management/test/unit/export/assets.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/assets.test.ts @@ -4,38 +4,28 @@ import { configHandler } from '@contentstack/cli-utilities'; import ExportAssets from '../../../src/export/assets'; import { CSAssetsExportAdapter } from '../../../src/export/base'; +import * as chunkedJsonReader from '../../../src/utils/chunked-json-reader'; +import * as retryModule from '../../../src/utils/retry'; import type { CSAssetsAPIConfig, LinkedWorkspace } from '../../../src/types/cs-assets-api'; import type { ExportContext } from '../../../src/types/export-types'; const foldersData = [{ uid: 'folder-1', name: 'Images' }]; -const assetsResponseWithItems = { - items: [ - { uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'image.png' }, - { uid: 'a2', url: 'https://cdn.example.com/a2.pdf', file_name: 'doc.pdf' }, - ], -}; -const emptyAssetsResponse = { items: [] as any[] }; +const assetItems = [ + { uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'image.png' }, + { uid: 'a2', url: 'https://cdn.example.com/a2.pdf', file_name: 'doc.pdf' }, +]; +const ASSET_META_KEYS = ['uid', 'url', 'filename', 'file_name', 'parent_uid']; describe('ExportAssets', () => { - const apiConfig: CSAssetsAPIConfig = { - baseURL: 'https://am.example.com', - headers: { organization_uid: 'org-1' }, - }; - - const exportContext: ExportContext = { - spacesRootPath: '/tmp/export/spaces', - }; - - const workspace: LinkedWorkspace = { - uid: 'ws-1', - space_uid: 'space-uid-1', - is_default: true, - }; - + const apiConfig: CSAssetsAPIConfig = { baseURL: 'https://am.example.com', headers: { organization_uid: 'org-1' } }; + const exportContext: ExportContext = { spacesRootPath: '/tmp/export/spaces' }; + const workspace: LinkedWorkspace = { uid: 'ws-1', space_uid: 'space-uid-1', is_default: true }; const spaceDir = '/tmp/export/spaces/space-uid-1'; let fetchStub: sinon.SinonStub; + let writerStub: { writeIntoFile: sinon.SinonStub; completeFile: sinon.SinonStub }; + let createWriterStub: sinon.SinonStub; const makeFetchResponse = () => { const webStream = new ReadableStream({ @@ -47,11 +37,39 @@ describe('ExportAssets', () => { return { ok: true, status: 200, body: webStream }; }; + /** + * Wire the streaming flow without real pagination or disk: `streamWorkspaceAssets` feeds `items` + * through the `onPage` sink, and the download read-back (`forEachChunkedJsonStore`) yields the + * same `items` back as one chunk (or signals empty). + */ + const wireStreaming = (items: Array>) => { + sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData as any); + sinon + .stub(ExportAssets.prototype, 'streamWorkspaceAssets') + .callsFake(async (_s: string, _ws: string | undefined, onPage: (i: unknown[]) => void | Promise) => { + await onPage(items); + return items.length; + }); + sinon + .stub(chunkedJsonReader, 'forEachChunkedJsonStore') + .callsFake(async (_base: string, _idx: string, opts: any, onChunk: (records: unknown[]) => Promise) => { + if (items.length === 0) { + opts.onEmptyIndexer(); + return; + } + await onChunk(items); + }); + }; + beforeEach(() => { sinon.stub(CSAssetsExportAdapter.prototype, 'init' as any).resolves(); - sinon.stub(CSAssetsExportAdapter.prototype, 'writeItemsToChunkedJson' as any).resolves(); sinon.stub(CSAssetsExportAdapter.prototype, 'tick' as any); sinon.stub(CSAssetsExportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsExportAdapter.prototype, 'writeEmptyChunkedJson' as any).resolves(); + writerStub = { writeIntoFile: sinon.stub(), completeFile: sinon.stub() }; + createWriterStub = sinon.stub(CSAssetsExportAdapter.prototype, 'createChunkedJsonWriter' as any).returns(writerStub); + // Run the retry wrapper inline (single attempt, no backoff) so tests don't wait on real delays. + sinon.stub(retryModule, 'withRetry').callsFake(async (fn: () => Promise) => fn()); fetchStub = sinon.stub(globalThis, 'fetch'); }); @@ -59,7 +77,7 @@ describe('ExportAssets', () => { sinon.restore(); }); - describe('start method', () => { + describe('concurrency config', () => { it('should use fallback download concurrency when not configured', () => { const exporter = new ExportAssets(apiConfig, exportContext); expect((exporter as any).downloadAssetsBatchConcurrency).to.equal(5); @@ -69,38 +87,45 @@ describe('ExportAssets', () => { const exporter = new ExportAssets(apiConfig, { ...exportContext, downloadAssetsConcurrency: 2 }); expect((exporter as any).downloadAssetsBatchConcurrency).to.equal(2); }); + }); - it('should fetch folders and assets using the workspace space_uid', async () => { - const foldersStub = sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - const assetsStub = sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(emptyAssetsResponse); - + describe('start method', () => { + it('should fetch folders and stream assets using the workspace space_uid', async () => { + wireStreaming([]); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); + const foldersStub = ExportAssets.prototype.getWorkspaceFolders as sinon.SinonStub; + const streamStub = ExportAssets.prototype.streamWorkspaceAssets as sinon.SinonStub; expect(foldersStub.firstCall.args[0]).to.equal(workspace.space_uid); - expect(assetsStub.firstCall.args[0]).to.equal(workspace.space_uid); + expect(streamStub.firstCall.args[0]).to.equal(workspace.space_uid); }); - it('should write chunked assets metadata with correct args', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + it('should stream asset metadata into a chunked-JSON writer', async () => { + wireStreaming(assetItems); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); - const writeStub = (CSAssetsExportAdapter.prototype as any).writeItemsToChunkedJson as sinon.SinonStub; - const args = writeStub.firstCall.args; - expect(args[1]).to.equal('assets.json'); - expect(args[2]).to.equal('assets'); - expect(args[3]).to.deep.equal(['uid', 'url', 'filename', 'file_name', 'parent_uid']); - expect(args[4]).to.have.length(2); + expect(createWriterStub.firstCall.args[1]).to.equal('assets.json'); + expect(createWriterStub.firstCall.args[2]).to.equal('assets'); + expect(createWriterStub.firstCall.args[3]).to.deep.equal(ASSET_META_KEYS); + expect(writerStub.writeIntoFile.firstCall.args[0]).to.have.length(2); + expect(writerStub.completeFile.calledOnceWith(true)).to.be.true; }); - it('should not attempt any downloads when the asset list is empty', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(emptyAssetsResponse); + it('should write an empty index (no writer) when there are no assets', async () => { + wireStreaming([]); + const exporter = new ExportAssets(apiConfig, exportContext); + await exporter.start(workspace, spaceDir); + expect(createWriterStub.called).to.be.false; + expect((CSAssetsExportAdapter.prototype as any).writeEmptyChunkedJson.calledOnce).to.be.true; + }); + + it('should not attempt any downloads when the asset list is empty', async () => { + wireStreaming([]); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); @@ -110,9 +135,8 @@ describe('ExportAssets', () => { expect(assetTicks).to.have.length(0); }); - it('should tick per failed asset with success=false and the error message on download failure', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + it('should tick per failed asset with success=false and the error on download failure', async () => { + wireStreaming(assetItems); fetchStub.rejects(new Error('network failure')); const exporter = new ExportAssets(apiConfig, exportContext); @@ -120,17 +144,15 @@ describe('ExportAssets', () => { const tickStub = (CSAssetsExportAdapter.prototype as any).tick as sinon.SinonStub; const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); - // Per-asset tick: one failure entry per attempted download. expect(assetTicks.length).to.be.greaterThan(0); for (const t of assetTicks) { expect(t.args[0]).to.be.false; - expect(t.args[2]).to.equal('network failure'); + expect(String(t.args[2])).to.include('network failure'); } }); it('should tick per asset with success=true and null error on successful downloads', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsResponseWithItems); + wireStreaming(assetItems); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); @@ -138,8 +160,7 @@ describe('ExportAssets', () => { const tickStub = (CSAssetsExportAdapter.prototype as any).tick as sinon.SinonStub; const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); - // One successful tick per asset in the workspace. - expect(assetTicks).to.have.length(assetsResponseWithItems.items.length); + expect(assetTicks).to.have.length(assetItems.length); for (const t of assetTicks) { expect(t.args[0]).to.be.true; expect(t.args[2]).to.be.null; @@ -147,16 +168,11 @@ describe('ExportAssets', () => { }); it('should skip assets that have neither a url nor a uid', async () => { - const incompleteAssets = { - items: [ - { uid: 'a1', url: null as any }, - { url: 'https://cdn.example.com/a2.png', filename: 'img.png' }, - { uid: null as any, url: null as any }, - ], - }; - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(incompleteAssets); - + wireStreaming([ + { uid: 'a1', url: null }, + { url: 'https://cdn.example.com/a2.png', filename: 'img.png' }, + { uid: null, url: null }, + ] as any); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); @@ -164,11 +180,7 @@ describe('ExportAssets', () => { }); it('should process assets that have _uid instead of uid without skipping them', async () => { - const assetsWithUnderscoreUid = { - items: [{ _uid: 'a-uid', url: 'https://cdn.example.com/a.png', filename: 'a.png' }], - }; - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsWithUnderscoreUid); + wireStreaming([{ _uid: 'a-uid', url: 'https://cdn.example.com/a.png', filename: 'a.png' }] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); @@ -179,69 +191,47 @@ describe('ExportAssets', () => { const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); expect(assetTicks).to.have.length(1); expect(assetTicks[0].args[0]).to.be.true; - expect(assetTicks[0].args[2]).to.be.null; }); it('should download assets that use file_name, and fall back to "asset" when both names are absent', async () => { - const assetsNoFilename = { - items: [ - { uid: 'a1', url: 'https://cdn.example.com/a1.pdf', file_name: 'named.pdf' }, - { uid: 'a2', url: 'https://cdn.example.com/a2.bin' }, - ], - }; - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves(assetsNoFilename); + wireStreaming([ + { uid: 'a1', url: 'https://cdn.example.com/a1.pdf', file_name: 'named.pdf' }, + { uid: 'a2', url: 'https://cdn.example.com/a2.bin' }, + ] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); const exporter = new ExportAssets(apiConfig, exportContext); await exporter.start(workspace, spaceDir); expect(fetchStub.callCount).to.equal(2); - expect(fetchStub.firstCall.args[0]).to.equal('https://cdn.example.com/a1.pdf'); - expect(fetchStub.secondCall.args[0]).to.equal('https://cdn.example.com/a2.bin'); - const tickStub = (CSAssetsExportAdapter.prototype as any).tick as sinon.SinonStub; - const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); - expect(assetTicks).to.have.length(2); - for (const t of assetTicks) expect(t.args[0]).to.be.true; + const urls = fetchStub.getCalls().map((c) => c.args[0]).sort(); + expect(urls).to.deep.equal(['https://cdn.example.com/a1.pdf', 'https://cdn.example.com/a2.bin']); }); it('should append authtoken to URL when securedAssets is true', async () => { sinon.stub(configHandler, 'get').returns('my-auth-token'); - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); - const securedContext: typeof exportContext = { ...exportContext, securedAssets: true }; - const exporter = new ExportAssets(apiConfig, securedContext); + const exporter = new ExportAssets(apiConfig, { ...exportContext, securedAssets: true }); await exporter.start(workspace, spaceDir); - const downloadUrl = fetchStub.firstCall.args[0] as string; - expect(downloadUrl).to.include('authtoken=my-auth-token'); + expect(String(fetchStub.firstCall.args[0])).to.include('authtoken=my-auth-token'); }); it('should use "&" separator when URL already contains "?"', async () => { sinon.stub(configHandler, 'get').returns('my-token'); - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1?v=1', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1?v=1', filename: 'img.png' }] as any); fetchStub.callsFake(async () => makeFetchResponse() as any); - const securedContext: typeof exportContext = { ...exportContext, securedAssets: true }; - const exporter = new ExportAssets(apiConfig, securedContext); + const exporter = new ExportAssets(apiConfig, { ...exportContext, securedAssets: true }); await exporter.start(workspace, spaceDir); - const downloadUrl = fetchStub.firstCall.args[0] as string; - expect(downloadUrl).to.include('?v=1&authtoken='); + expect(String(fetchStub.firstCall.args[0])).to.include('?v=1&authtoken='); }); it('should tick with success=false and the HTTP status code on non-ok response', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }] as any); fetchStub.resolves({ ok: false, status: 403, body: null } as any); const exporter = new ExportAssets(apiConfig, exportContext); @@ -251,14 +241,11 @@ describe('ExportAssets', () => { const assetTicks = tickStub.getCalls().filter((c) => String(c.args[1]).startsWith('asset:')); expect(assetTicks).to.have.length(1); expect(assetTicks[0].args[0]).to.be.false; - expect(assetTicks[0].args[2]).to.include('403'); + expect(String(assetTicks[0].args[2])).to.include('403'); }); it('should tick with success=false and "No response body" when body is null', async () => { - sinon.stub(ExportAssets.prototype, 'getWorkspaceFolders').resolves(foldersData); - sinon.stub(ExportAssets.prototype, 'getWorkspaceAssets').resolves({ - items: [{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }], - }); + wireStreaming([{ uid: 'a1', url: 'https://cdn.example.com/a1.png', filename: 'img.png' }] as any); fetchStub.resolves({ ok: true, status: 200, body: null } as any); const exporter = new ExportAssets(apiConfig, exportContext); diff --git a/packages/contentstack-asset-management/test/unit/export/base.test.ts b/packages/contentstack-asset-management/test/unit/export/base.test.ts index 08993eddf..22f41d67e 100644 --- a/packages/contentstack-asset-management/test/unit/export/base.test.ts +++ b/packages/contentstack-asset-management/test/unit/export/base.test.ts @@ -35,6 +35,12 @@ class TestAdapter extends CSAssetsExportAdapter { public get spacesRootPathPublic() { return this.spacesRootPath; } + public get apiPageSizePublic() { + return this.apiPageSize; + } + public get apiFetchConcurrencyPublic() { + return this.apiFetchConcurrency; + } } describe('CSAssetsExportAdapter (base)', () => { @@ -192,6 +198,30 @@ describe('CSAssetsExportAdapter (base)', () => { }); }); + describe('apiPageSize', () => { + it('should return FALLBACK_AM_API_PAGE_SIZE (100) when pageSize is not set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.apiPageSizePublic).to.equal(100); + }); + + it('should return the configured pageSize when set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, { ...exportContext, pageSize: 50 }); + expect(adapter.apiPageSizePublic).to.equal(50); + }); + }); + + describe('apiFetchConcurrency', () => { + it('should return FALLBACK_AM_API_FETCH_CONCURRENCY (5) when fetchConcurrency is not set', () => { + const adapter = new TestAdapter(apiConfig, exportContext); + expect(adapter.apiFetchConcurrencyPublic).to.equal(5); + }); + + it('should return the configured fetchConcurrency when set in exportContext', () => { + const adapter = new TestAdapter(apiConfig, { ...exportContext, fetchConcurrency: 10 }); + expect(adapter.apiFetchConcurrencyPublic).to.equal(10); + }); + }); + describe('writeItemsToChunkedJson', () => { it('should write {} to an empty file when items array is empty', async () => { const os = require('node:os'); diff --git a/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts b/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts new file mode 100644 index 000000000..c3252526d --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/asset-types.test.ts @@ -0,0 +1,186 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportAssetTypes from '../../../src/import/asset-types'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportAssetTypes', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'getAssetTypesDir' as any).returns('/tmp/import/spaces/asset_types'); + }); + + afterEach(() => sinon.restore()); + + const stubExistingAssetTypes = (assetTypes: any[]) => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceAssetTypes' as any) + .resolves({ asset_types: assetTypes }); + }; + + const stubChunks = (records: Record[]) => { + const indexer = records.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (records.length > 0) { + const chunk = Object.fromEntries(records.map((r) => [(r.uid as string), r])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + }; + + describe('when index file does not exist', () => { + it('ticks once and returns without calling createAssetType', async () => { + sinon.stub(require('node:fs'), 'existsSync').returns(false); + stubExistingAssetTypes([]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + }); + + describe('when asset types exist in the export', () => { + beforeEach(() => { + sinon.stub(require('node:fs'), 'existsSync').returns(true); + }); + + it('creates a new asset type that does not exist in the target org', async () => { + const newType = { uid: 'type-new', label: 'New Type' }; + stubExistingAssetTypes([]); + stubChunks([newType]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + const payload = createStub.firstCall.args[0]; + expect(payload.label).to.equal('New Type'); + }); + + it('skips asset types with is_system=true', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'sys-type', is_system: true, label: 'System Type' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + const tickArgs = tickStub.lastCall.args[1] as string; + expect(tickArgs).to.include('skipped'); + }); + + it('skips (no create) when uid already exists in target with matching definition', async () => { + const existing = { uid: 'type-1', label: 'Type One', created_at: '2024-01-01' }; + const exported = { uid: 'type-1', label: 'Type One', created_at: '2024-01-01' }; + stubExistingAssetTypes([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('skips (no create) when uid exists with a different definition in target', async () => { + const existing = { uid: 'type-1', label: 'Old Label' }; + const exported = { uid: 'type-1', label: 'New Label' }; + stubExistingAssetTypes([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('strips invalid keys (created_at, updated_at, is_system) from the POST payload', async () => { + const exported = { + uid: 'type-clean', + label: 'Clean Type', + created_at: '2024-01-01', + updated_at: '2024-06-01', + is_system: false, + created_by: 'user-1', + updated_by: 'user-2', + }; + stubExistingAssetTypes([]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const payload = createStub.firstCall.args[0]; + expect(payload).to.not.have.property('created_at'); + expect(payload).to.not.have.property('updated_at'); + expect(payload).to.not.have.property('is_system'); + expect(payload).to.not.have.property('created_by'); + expect(payload).to.not.have.property('updated_by'); + expect(payload.label).to.equal('Clean Type'); + }); + + it('handles createAssetType failure: increments failure count, final tick reflects failure', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'type-bad', label: 'Bad Type' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).rejects(new Error('API error')); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(false); + expect(lastTickArgs[1]).to.include('1 failed'); + }); + + it('handles getWorkspaceAssetTypes failure: proceeds as if no existing types', async () => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceAssetTypes' as any) + .rejects(new Error('API unavailable')); + stubChunks([{ uid: 'type-new', label: 'New Type' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + }); + + it('final tick is success=true when all creates succeed', async () => { + stubExistingAssetTypes([]); + stubChunks([{ uid: 'type-ok', label: 'OK Type' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createAssetType' as any).resolves(); + + const importer = new ImportAssetTypes(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(true); + expect(lastTickArgs[1]).to.include('1 created'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/assets.test.ts b/packages/contentstack-asset-management/test/unit/import/assets.test.ts new file mode 100644 index 000000000..d60a758ae --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/assets.test.ts @@ -0,0 +1,239 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as os from 'os'; +import * as path from 'path'; +import * as fsReal from 'fs'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportAssets from '../../../src/import/assets'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportAssets', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + }); + + afterEach(() => sinon.restore()); + + const makeSpaceDir = () => { + const dir = path.join(os.tmpdir(), `am-test-${Date.now()}`); + fsReal.mkdirSync(path.join(dir, 'assets'), { recursive: true }); + return dir; + }; + + const stubAssetChunks = (assets: Record[]) => { + const indexer = assets.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (assets.length > 0) { + const chunk = Object.fromEntries(assets.map((a) => [(a.uid as string), a])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + sinon.stub(FsUtility.prototype, 'getPlainMeta').returns( + assets.length > 0 ? { 'chunk0': assets.map((a) => a.uid) } : {}, + ); + }; + + describe('buildIdentityMappersFromExport', () => { + it('returns empty maps when no assets.json index exists in spaceDir', async () => { + const spaceDir = makeSpaceDir(); + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.deep.equal({}); + expect(result.urlMap).to.deep.equal({}); + }); + + it('builds identity uid and url maps from chunked assets', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([ + { uid: 'asset-1', url: 'https://cdn.example.com/asset-1.png' }, + { uid: 'asset-2', url: 'https://cdn.example.com/asset-2.png' }, + ]); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.deep.equal({ 'asset-1': 'asset-1', 'asset-2': 'asset-2' }); + expect(result.urlMap).to.deep.equal({ + 'https://cdn.example.com/asset-1.png': 'https://cdn.example.com/asset-1.png', + 'https://cdn.example.com/asset-2.png': 'https://cdn.example.com/asset-2.png', + }); + }); + + it('handles assets with missing uid gracefully: only url is added to urlMap', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({ '0': true })); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves({ + 'asset-no-url': { uid: 'asset-no-url' }, + }), + })); + sinon.stub(FsUtility.prototype, 'getPlainMeta').returns({}); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.buildIdentityMappersFromExport(spaceDir); + + expect(result.uidMap).to.have.key('asset-no-url'); + expect(result.urlMap).to.deep.equal({}); + }); + }); + + describe('start', () => { + it('returns empty maps and ticks once for an empty space (no folders, no assets)', async () => { + const spaceDir = makeSpaceDir(); + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('new-space-uid', spaceDir); + + expect(result.uidMap).to.deep.equal({}); + expect(result.urlMap).to.deep.equal({}); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + + it('creates root-level folders and maps their uids', async () => { + const spaceDir = makeSpaceDir(); + const folders = [{ uid: 'folder-old', title: 'My Folder' }]; + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders }), + ); + stubAssetChunks([]); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({})); + const createFolderStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .resolves({ folder: { uid: 'folder-new' } }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + expect(createFolderStub.callCount).to.equal(1); + const createArgs = createFolderStub.firstCall.args; + expect(createArgs[0]).to.equal('space-uid'); + expect(createArgs[1].title).to.equal('My Folder'); + }); + + it('imports nested folders in multi-pass: child waits for parent to be created', async () => { + const spaceDir = makeSpaceDir(); + const folders = [ + { uid: 'child-folder', title: 'Child', parent_uid: 'parent-folder' }, + { uid: 'parent-folder', title: 'Parent' }, + ]; + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders }), + ); + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({})); + let callOrder: string[] = []; + const createFolderStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .callsFake(async (_spaceUid: string, payload: any) => { + callOrder.push(payload.title); + return { folder: { uid: `new-${payload.title.toLowerCase()}` } }; + }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + expect(createFolderStub.callCount).to.equal(2); + expect(callOrder[0]).to.equal('Parent'); + expect(callOrder[1]).to.equal('Child'); + }); + + it('uploads assets: calls uploadAsset and builds uidMap and urlMap', async () => { + const spaceDir = makeSpaceDir(); + const assetUid = 'asset-old-uid'; + const assetFilename = 'photo.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, assetFilename), 'fake-content'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, url: 'https://old-cdn.com/photo.png', filename: assetFilename }]); + + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any) + .resolves({ asset: { uid: 'asset-new-uid', url: 'https://new-cdn.com/photo.png' } }); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(uploadStub.callCount).to.equal(1); + expect(result.uidMap[assetUid]).to.equal('asset-new-uid'); + expect(result.urlMap['https://old-cdn.com/photo.png']).to.equal('https://new-cdn.com/photo.png'); + }); + + it('skips an asset and ticks false when the file is not found on disk', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: 'missing-asset', url: 'https://cdn.com/x.png', filename: 'x.png' }]); + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any).resolves(); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(uploadStub.callCount).to.equal(0); + expect(result.uidMap).to.deep.equal({}); + const failTick = tickStub.getCalls().find((c) => c.args[0] === false && c.args[2]); + expect(failTick).to.exist; + }); + + it('handles uploadAsset failure gracefully: continues, ticks false, omits from maps', async () => { + const spaceDir = makeSpaceDir(); + const assetUid = 'asset-fail'; + const filename = 'fail.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, filename), 'data'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, url: 'https://cdn.com/fail.png', filename }]); + + sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any).rejects(new Error('upload failed')); + + const importer = new ImportAssets(apiConfig, importContext); + const result = await importer.start('space-uid', spaceDir); + + expect(result.uidMap).to.deep.equal({}); + const failTick = tickStub.getCalls().find((c) => c.args[0] === false); + expect(failTick).to.exist; + }); + + it('maps asset parent_uid to the new folder uid when parent was imported', async () => { + const spaceDir = makeSpaceDir(); + fsReal.writeFileSync( + path.join(spaceDir, 'assets', 'folders.json'), + JSON.stringify({ folders: [{ uid: 'old-folder', title: 'Folder A' }] }), + ); + const assetUid = 'asset-in-folder'; + const filename = 'file.png'; + fsReal.mkdirSync(path.join(spaceDir, 'assets', 'files', assetUid), { recursive: true }); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'files', assetUid, filename), 'data'); + fsReal.writeFileSync(path.join(spaceDir, 'assets', 'assets.json'), '{}'); + stubAssetChunks([{ uid: assetUid, parent_uid: 'old-folder', filename }]); + + sinon.stub(CSAssetsImportAdapter.prototype, 'createFolder' as any) + .resolves({ folder: { uid: 'new-folder-uid' } }); + const uploadStub = sinon.stub(CSAssetsImportAdapter.prototype, 'uploadAsset' as any) + .resolves({ asset: { uid: 'new-asset-uid' } }); + + const importer = new ImportAssets(apiConfig, importContext); + await importer.start('space-uid', spaceDir); + + const uploadArgs = uploadStub.firstCall.args; + expect(uploadArgs[2].parent_uid).to.equal('new-folder-uid'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/base.test.ts b/packages/contentstack-asset-management/test/unit/import/base.test.ts new file mode 100644 index 000000000..ddbab234e --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/base.test.ts @@ -0,0 +1,224 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +class TestImportAdapter extends CSAssetsImportAdapter { + public callCreateNestedProgress(name: string) { return this.createNestedProgress(name); } + public callTick(success: boolean, name: string, error: string | null, processName?: string) { + return this.tick(success, name, error, processName); + } + public callUpdateStatus(msg: string, processName?: string) { return this.updateStatus(msg, processName); } + public callCompleteProcess(name: string, success: boolean) { return this.completeProcess(name, success); } + public get progressOrParentPublic() { return this.progressOrParent; } + public get spacesRootPathPublic() { return this.spacesRootPath; } + public get apiConcurrencyPublic() { return this.apiConcurrency; } + public get uploadBatchPublic() { return this.uploadAssetsBatchConcurrency; } + public get foldersBatchPublic() { return this.importFoldersBatchConcurrency; } + public getAssetTypesDirPublic() { return this.getAssetTypesDir(); } + public getFieldsDirPublic() { return this.getFieldsDir(); } +} + +describe('CSAssetsImportAdapter (base)', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + }); + afterEach(() => sinon.restore()); + + describe('setParentProgressManager / progressOrParent', () => { + it('returns null when no progress manager is set', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.progressOrParentPublic).to.be.null; + }); + + it('returns the parent manager after setParentProgressManager', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + expect(adapter.progressOrParentPublic).to.equal(fakeParent); + }); + + it('returns progressManager when parentProgressManager is not set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test-module'); + expect(adapter.progressOrParentPublic).to.equal(fakeProgress); + }); + }); + + describe('setProcessName', () => { + it('overrides the processName used in tick calls', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.setProcessName('custom-process'); + adapter.callTick(true, 'item', null); + expect(fakeParent.tick.firstCall.args[3]).to.equal('custom-process'); + }); + }); + + describe('createNestedProgress', () => { + it('creates a CLIProgressManager when no parent is set', () => { + sinon.stub(configHandler, 'get').returns({ showConsoleLogs: true }); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + const result = adapter.callCreateNestedProgress('my-module'); + expect(createNestedStub.firstCall.args[0]).to.equal('my-module'); + expect(result).to.equal(fakeProgress); + }); + + it('returns parent directly when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + const result = adapter.callCreateNestedProgress('ignored'); + expect(result).to.equal(fakeParent); + }); + + it('defaults showConsoleLogs to false when log config is missing', () => { + sinon.stub(configHandler, 'get').returns(null); + const fakeProgress = { tick: sinon.stub() } as any; + const createNestedStub = sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test'); + expect(createNestedStub.firstCall.args[1]).to.be.false; + }); + }); + + describe('tick', () => { + it('forwards success, itemName, error to progress manager tick', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callTick(true, 'my-item', 'some-error'); + expect(fakeParent.tick.firstCall.args[0]).to.equal(true); + expect(fakeParent.tick.firstCall.args[1]).to.equal('my-item'); + expect(fakeParent.tick.firstCall.args[2]).to.equal('some-error'); + }); + + it('uses explicit processName override when provided', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callTick(false, 'item', null, 'override-process'); + expect(fakeParent.tick.firstCall.args[3]).to.equal('override-process'); + }); + + it('does not throw when progressOrParent is null', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(() => adapter.callTick(true, 'item', null)).to.not.throw(); + }); + }); + + describe('updateStatus', () => { + it('forwards status message to progress manager', () => { + const fakeParent = { tick: sinon.stub(), updateStatus: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callUpdateStatus('Importing...'); + expect(fakeParent.updateStatus.firstCall.args[0]).to.equal('Importing...'); + }); + + it('does not throw when progressOrParent is null', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(() => adapter.callUpdateStatus('msg')).to.not.throw(); + }); + }); + + describe('completeProcess', () => { + it('calls completeProcess on progressManager when no parent is set', () => { + sinon.stub(configHandler, 'get').returns({}); + const fakeProgress = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress); + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.callCreateNestedProgress('test'); + adapter.callCompleteProcess('test-process', true); + expect(fakeProgress.completeProcess.firstCall.args).to.deep.equal(['test-process', true]); + }); + + it('does NOT call completeProcess when parentProgressManager is set', () => { + const fakeParent = { tick: sinon.stub(), completeProcess: sinon.stub() } as any; + const adapter = new TestImportAdapter(apiConfig, importContext); + adapter.setParentProgressManager(fakeParent); + adapter.callCompleteProcess('test-process', true); + expect(fakeParent.completeProcess.callCount).to.equal(0); + }); + }); + + describe('path and concurrency getters', () => { + it('spacesRootPath returns the value from importContext', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.spacesRootPathPublic).to.equal('/tmp/import/spaces'); + }); + + it('apiConcurrency defaults to FALLBACK_AM_API_CONCURRENCY (5) when not set', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + expect(adapter.apiConcurrencyPublic).to.equal(5); + }); + + it('apiConcurrency uses importContext.apiConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 10 }); + expect(adapter.apiConcurrencyPublic).to.equal(10); + }); + + it('uploadAssetsBatchConcurrency falls back to apiConcurrency when uploadAssetsConcurrency not set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 8 }); + expect(adapter.uploadBatchPublic).to.equal(8); + }); + + it('uploadAssetsBatchConcurrency uses uploadAssetsConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, uploadAssetsConcurrency: 3 }); + expect(adapter.uploadBatchPublic).to.equal(3); + }); + + it('importFoldersBatchConcurrency falls back to apiConcurrency when not set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, apiConcurrency: 6 }); + expect(adapter.foldersBatchPublic).to.equal(6); + }); + + it('importFoldersBatchConcurrency uses importFoldersConcurrency when set', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, importFoldersConcurrency: 2 }); + expect(adapter.foldersBatchPublic).to.equal(2); + }); + + it('getAssetTypesDir defaults to spacesRootPath/asset_types', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + const expected = require('node:path').join('/tmp/import/spaces', 'asset_types'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('getAssetTypesDir uses custom assetTypesDir when set in importContext', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, assetTypesDir: 'custom_at' }); + const expected = require('node:path').join('/tmp/import/spaces', 'custom_at'); + expect(adapter.getAssetTypesDirPublic()).to.equal(expected); + }); + + it('getFieldsDir defaults to spacesRootPath/fields', () => { + const adapter = new TestImportAdapter(apiConfig, importContext); + const expected = require('node:path').join('/tmp/import/spaces', 'fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + + it('getFieldsDir uses custom fieldsDir when set in importContext', () => { + const adapter = new TestImportAdapter(apiConfig, { ...importContext, fieldsDir: 'custom_fields' }); + const expected = require('node:path').join('/tmp/import/spaces', 'custom_fields'); + expect(adapter.getFieldsDirPublic()).to.equal(expected); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/fields.test.ts b/packages/contentstack-asset-management/test/unit/import/fields.test.ts new file mode 100644 index 000000000..c45ef17c8 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/import/fields.test.ts @@ -0,0 +1,186 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import ImportFields from '../../../src/import/fields'; +import { CSAssetsImportAdapter } from '../../../src/import/base'; +import type { CSAssetsAPIConfig, ImportContext } from '../../../src/types/cs-assets-api'; + +describe('ImportFields', () => { + const apiConfig: CSAssetsAPIConfig = { + baseURL: 'https://am.example.com', + headers: { organization_uid: 'org-1' }, + }; + const importContext: ImportContext = { + spacesRootPath: '/tmp/import/spaces', + apiKey: 'api-key-1', + host: 'https://api.contentstack.io/v3', + org_uid: 'org-1', + }; + + let tickStub: sinon.SinonStub; + + beforeEach(() => { + sinon.stub(CSAssetsImportAdapter.prototype, 'init' as any).resolves(); + tickStub = sinon.stub(CSAssetsImportAdapter.prototype, 'tick' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'updateStatus' as any); + sinon.stub(CSAssetsImportAdapter.prototype, 'getFieldsDir' as any).returns('/tmp/import/spaces/fields'); + }); + + afterEach(() => sinon.restore()); + + const stubExistingFields = (fields: any[]) => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceFields' as any) + .resolves({ fields }); + }; + + const stubChunks = (records: Record[]) => { + const indexer = records.length > 0 ? { '0': true } : {}; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => indexer); + if (records.length > 0) { + const chunk = Object.fromEntries(records.map((r) => [(r.uid as string), r])); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves(chunk), + })); + } + }; + + describe('when index file does not exist', () => { + it('ticks once and returns without calling createField', async () => { + sinon.stub(require('node:fs'), 'existsSync').returns(false); + stubExistingFields([]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.callCount).to.equal(1); + expect(tickStub.firstCall.args[0]).to.equal(true); + }); + }); + + describe('when fields exist in the export', () => { + beforeEach(() => { + sinon.stub(require('node:fs'), 'existsSync').returns(true); + }); + + it('creates a new field that does not exist in the target org', async () => { + const newField = { uid: 'field-new', label: 'New Field', type: 'text' }; + stubExistingFields([]); + stubChunks([newField]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + const payload = createStub.firstCall.args[0]; + expect(payload.label).to.equal('New Field'); + }); + + it('skips fields with is_system=true', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'sys-field', is_system: true, label: 'System Field' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('silently skips (no create) when uid exists with matching definition after stripping invalid keys', async () => { + const existing = { uid: 'field-1', label: 'Field One', created_at: '2024-01-01', asset_types_count: 3 }; + const exported = { uid: 'field-1', label: 'Field One', created_at: '2024-01-01', asset_types_count: 5 }; + stubExistingFields([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('skips (no create) when uid exists with a different definition', async () => { + const existing = { uid: 'field-1', label: 'Old Label', type: 'text' }; + const exported = { uid: 'field-1', label: 'New Label', type: 'text' }; + stubExistingFields([existing]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(0); + expect(tickStub.lastCall.args[1]).to.include('skipped'); + }); + + it('strips invalid keys (created_at, updated_at, is_system, asset_types_count) from POST payload', async () => { + const exported = { + uid: 'field-clean', + label: 'Clean Field', + created_at: '2024-01-01', + updated_at: '2024-06-01', + is_system: false, + asset_types_count: 10, + created_by: 'user-1', + updated_by: 'user-2', + }; + stubExistingFields([]); + stubChunks([exported]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const payload = createStub.firstCall.args[0]; + expect(payload).to.not.have.property('created_at'); + expect(payload).to.not.have.property('updated_at'); + expect(payload).to.not.have.property('is_system'); + expect(payload).to.not.have.property('asset_types_count'); + expect(payload.label).to.equal('Clean Field'); + }); + + it('handles createField failure: final tick reflects failure count', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'field-bad', label: 'Bad Field' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).rejects(new Error('API error')); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(false); + expect(lastTickArgs[1]).to.include('1 failed'); + }); + + it('handles getWorkspaceFields failure: proceeds as if no existing fields', async () => { + sinon.stub(CSAssetsImportAdapter.prototype, 'getWorkspaceFields' as any) + .rejects(new Error('API unavailable')); + stubChunks([{ uid: 'field-new', label: 'New Field' }]); + const createStub = sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + expect(createStub.callCount).to.equal(1); + }); + + it('final tick is success=true when all creates succeed and none fail', async () => { + stubExistingFields([]); + stubChunks([{ uid: 'field-ok', label: 'OK Field' }]); + sinon.stub(CSAssetsImportAdapter.prototype, 'createField' as any).resolves(); + + const importer = new ImportFields(apiConfig, importContext); + await importer.start(); + + const lastTickArgs = tickStub.lastCall.args; + expect(lastTickArgs[0]).to.equal(true); + expect(lastTickArgs[1]).to.include('1 created'); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/import/spaces.test.ts b/packages/contentstack-asset-management/test/unit/import/spaces.test.ts index 5cff18b66..d77f7e943 100644 --- a/packages/contentstack-asset-management/test/unit/import/spaces.test.ts +++ b/packages/contentstack-asset-management/test/unit/import/spaces.test.ts @@ -30,7 +30,10 @@ describe('ImportSpaces', () => { }; beforeEach(() => { - sinon.stub(configHandler, 'get').returns({ showConsoleLogs: false }); + sinon.stub(configHandler, 'get').callsFake((key: string) => { + if (key === 'log') return { showConsoleLogs: false }; + return undefined; + }); sinon.stub(CLIProgressManager, 'createNested').returns(fakeProgress as any); // init and listSpaces live on AssetManagementAdapter (the common base). // Stubbing the base once covers both the adapter used for listSpaces and ImportWorkspace. @@ -173,4 +176,178 @@ describe('ImportSpaces', () => { expect(result.spaceUidMap).to.deep.equal({}); }); }); + + describe('bootstrap failure', () => { + it('should mark all space rows as failed and re-throw when ImportFields throws', async () => { + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + (ImportFields.prototype.start as sinon.SinonStub).rejects(new Error('fields-bootstrap-error')); + + const importer = new ImportSpaces(baseOptions); + try { + await importer.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('fields-bootstrap-error'); + } + + const completeCalls = fakeProgress.completeProcess.getCalls().map((c) => c.args); + expect(completeCalls).to.deep.include([PROCESS_NAMES.AM_IMPORT_FIELDS, false]); + }); + + it('should mark all space rows as failed and re-throw when ImportAssetTypes throws', async () => { + stubSpaceDirs(['am-space-1']); + (ImportAssetTypes.prototype.start as sinon.SinonStub).rejects(new Error('at-bootstrap-error')); + + const importer = new ImportSpaces(baseOptions); + try { + await importer.start(); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.equal('at-bootstrap-error'); + } + + const completeCalls = fakeProgress.completeProcess.getCalls().map((c) => c.args); + expect(completeCalls).to.deep.include([PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, false]); + }); + }); + + describe('per-space failure resilience', () => { + it('should continue importing remaining spaces when one space fails', async () => { + stubSpaceDirs(['am-space-1', 'am-space-2']); + const startStub = sinon.stub(ImportWorkspace.prototype, 'start'); + startStub.onFirstCall().rejects(new Error('space-1-error')); + startStub.onSecondCall().resolves({ + oldSpaceUid: 'am-space-2', newSpaceUid: 'new-space-2', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(startStub.callCount).to.equal(2); + expect(result.spaceMappings).to.have.lengthOf(1); + expect(result.spaceMappings[0].oldSpaceUid).to.equal('am-space-2'); + }); + }); + + describe('backupDir mapper file writing', () => { + it('should write uid, url, and space-uid mapping files when backupDir is set', async () => { + const os = require('node:os'); + const path = require('node:path'); + const fsReal = require('node:fs'); + const tmpDir = path.join(os.tmpdir(), `import-spaces-backup-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space-1', workspaceUid: 'main', + isDefault: false, + uidMap: { 'old-uid': 'new-uid' }, + urlMap: { 'old-url': 'new-url' }, + }); + + const options: ImportSpacesOptions = { ...baseOptions, backupDir: tmpDir }; + const importer = new ImportSpaces(options); + await importer.start(); + + const mapperDir = path.join(tmpDir, 'mapper', 'assets'); + expect(fsReal.existsSync(path.join(mapperDir, 'uid-mapping.json'))).to.be.true; + expect(fsReal.existsSync(path.join(mapperDir, 'url-mapping.json'))).to.be.true; + expect(fsReal.existsSync(path.join(mapperDir, 'space-uid-mapping.json'))).to.be.true; + + const uidMap = JSON.parse(fsReal.readFileSync(path.join(mapperDir, 'uid-mapping.json'), 'utf8')); + expect(uidMap).to.deep.equal({ 'old-uid': 'new-uid' }); + }); + }); + + describe('listSpaces error handling and uid filtering', () => { + it('should pass existing org space uids to ImportWorkspace when listSpaces returns spaces', async () => { + (CSAssetsAdapter.prototype.listSpaces as sinon.SinonStub).resolves({ spaces: [{ uid: 'org-space-uid' }] }); + stubSpaceDirs(['am-space-1']); + const startStub = sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-space', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + await importer.start(); + + expect(startStub.callCount).to.equal(1); + const existingSpaceUids: Set = startStub.firstCall.args[2]; + expect(existingSpaceUids.has('org-space-uid')).to.be.true; + }); + + it('should continue (disable reuse-by-uid) when listSpaces throws', async () => { + (CSAssetsAdapter.prototype.listSpaces as sinon.SinonStub).rejects(new Error('network error')); + stubSpaceDirs(['am-space-1']); + sinon.stub(ImportWorkspace.prototype, 'start').resolves({ + oldSpaceUid: 'am-space-1', newSpaceUid: 'new-uid', workspaceUid: 'main', + isDefault: false, uidMap: {}, urlMap: {}, + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.have.lengthOf(1); + }); + + it('should return false for a directory entry when statSync throws', async () => { + const fsMock = require('node:fs'); + const pResolve = require('node:path').resolve; + const join = require('node:path').join; + const spacesRoot = pResolve('/tmp/import', 'spaces'); + const origStatSync = fsMock.statSync.bind(fsMock); + sinon.stub(fsMock, 'readdirSync').returns(['am-bad-entry'] as any); + sinon.stub(fsMock, 'statSync').callsFake((p: string) => { + if (p === join(spacesRoot, 'am-bad-entry')) throw new Error('permission denied'); + return origStatSync(p); + }); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.deep.equal([]); + }); + + it('should log warning and return empty dirs when readdirSync throws', async () => { + const fsMock = require('node:fs'); + const pResolve = require('node:path').resolve; + const spacesRoot = pResolve('/tmp/import', 'spaces'); + const origReaddir = fsMock.readdirSync.bind(fsMock); + sinon.stub(fsMock, 'readdirSync').callsFake((p: string) => { + if (p === spacesRoot) throw new Error('ENOENT: no such file or directory'); + return origReaddir(p); + }); + sinon.stub(fsMock, 'statSync').returns({ isDirectory: () => true } as any); + + const importer = new ImportSpaces(baseOptions); + const result = await importer.start(); + + expect(result.spaceMappings).to.deep.equal([]); + }); + }); + + describe('setParentProgressManager', () => { + it('should use parent progress manager instead of creating a new CLIProgressManager', async () => { + const fakeParent = { + addProcess: sinon.stub().returnsThis(), + startProcess: sinon.stub().returnsThis(), + updateStatus: sinon.stub().returnsThis(), + tick: sinon.stub(), + completeProcess: sinon.stub(), + }; + stubSpaceDirs([]); + + const importer = new ImportSpaces(baseOptions); + importer.setParentProgressManager(fakeParent as any); + await importer.start(); + + expect((CLIProgressManager.createNested as sinon.SinonStub).callCount).to.equal(0); + expect(fakeParent.addProcess.callCount).to.be.greaterThan(0); + }); + }); }); diff --git a/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts index 7d5af5090..87e5dbe6a 100644 --- a/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts +++ b/packages/contentstack-asset-management/test/unit/query-export/cs-assets-query-exporter.test.ts @@ -10,7 +10,7 @@ import ExportAssetTypes from '../../../src/export/asset-types'; import ExportFields from '../../../src/export/fields'; import { CSAssetsExportAdapter } from '../../../src/export/base'; import { CSAssetsAdapter } from '../../../src/utils/cs-assets-api-adapter'; -import * as concurrentBatch from '../../../src/utils/concurrent-batch'; +import * as retryModule from '../../../src/utils/retry'; import type { CsAssetsQueryExportOptions } from '../../../src/types/cs-assets-api'; @@ -46,11 +46,32 @@ describe('CsAssetsQueryExporter', () => { ], }); sinon.stub(CSAssetsExportAdapter.prototype as any, 'writeItemsToChunkedJson').resolves(); - sinon.stub(concurrentBatch, 'runInBatches').callsFake(async (items, _concurrency, handler) => { - for (let i = 0; i < items.length; i++) { - await handler(items[i], i); + // Downloads now run through makeConcurrentCall; fake it by invoking the + // promisifyHandler synchronously over each element of every apiBatch. + sinon.stub(CSAssetsAdapter.prototype, 'makeConcurrentCall').callsFake(async (env: any, handler: any) => { + const batches = env?.apiBatches ?? []; + for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) { + for (let index = 0; index < batches[batchIndex].length; index++) { + if (handler) await handler({ index, batchIndex, isLastRequest: false }); + } } }); + // Run the download retry wrapper inline (single attempt, no backoff) and serve a fake binary + // so download attempts don't hit the network or wait on real retry delays. + sinon.stub(retryModule, 'withRetry').callsFake(async (fn: () => Promise) => fn()); + sinon.stub(globalThis, 'fetch').callsFake( + async () => + ({ + ok: true, + status: 200, + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('x')); + controller.close(); + }, + }), + }) as any, + ); }); afterEach(() => { diff --git a/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts b/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts new file mode 100644 index 000000000..309b24776 --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/chunked-json-reader.test.ts @@ -0,0 +1,143 @@ +import { expect } from 'chai'; +import sinon from 'sinon'; +import * as os from 'os'; +import * as fsReal from 'fs'; +import * as path from 'path'; +import { FsUtility } from '@contentstack/cli-utilities'; + +import { forEachChunkedJsonStore, forEachChunkRecordsFromFs } from '../../../src/utils/chunked-json-reader'; + +describe('chunked-json-reader', () => { + afterEach(() => sinon.restore()); + + const makeFakeFs = (indexer: Record, chunks: unknown[]): FsUtility => { + let idx = 0; + return { + indexFileContent: indexer, + readChunkFiles: { next: async () => chunks[idx++] ?? null }, + getPlainMeta: () => ({}), + } as unknown as FsUtility; + }; + + describe('forEachChunkRecordsFromFs', () => { + it('does nothing when indexer is empty', async () => { + const onChunk = sinon.stub(); + await forEachChunkRecordsFromFs(makeFakeFs({}, []), { chunkReadLogLabel: 'test' }, onChunk); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onChunk with Object.values of each chunk record', async () => { + const r1 = { uid: 'uid-1', url: 'https://a.com' }; + const r2 = { uid: 'uid-2', url: 'https://b.com' }; + const collected: unknown[] = []; + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true }, [{ 'uid-1': r1, 'uid-2': r2 }]), + { chunkReadLogLabel: 'assets' }, + async (records) => { collected.push(...records); }, + ); + expect(collected).to.deep.equal([r1, r2]); + }); + + it('processes multiple chunks in order', async () => { + const order: string[] = []; + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true, '1': true }, [ + { 'uid-A': { uid: 'uid-A' } }, + { 'uid-B': { uid: 'uid-B' } }, + ]), + { chunkReadLogLabel: 'test' }, + async (records: any[]) => { order.push(...records.map((r) => r.uid)); }, + ); + expect(order).to.deep.equal(['uid-A', 'uid-B']); + }); + + it('skips a chunk when readChunkFiles.next() rejects', async () => { + const onChunk = sinon.stub(); + const fakeFs = { + indexFileContent: { '0': true }, + readChunkFiles: { next: sinon.stub().rejects(new Error('disk error')) }, + } as unknown as FsUtility; + await forEachChunkRecordsFromFs(fakeFs, { chunkReadLogLabel: 'test' }, onChunk); + expect(onChunk.callCount).to.equal(0); + }); + + it('skips null chunks returned by readChunkFiles', async () => { + const onChunk = sinon.stub(); + await forEachChunkRecordsFromFs( + makeFakeFs({ '0': true }, [null]), + { chunkReadLogLabel: 'test' }, + onChunk, + ); + expect(onChunk.callCount).to.equal(0); + }); + }); + + describe('forEachChunkedJsonStore', () => { + it('calls onOpenError and does not call onEmptyIndexer or onChunk when FsUtility constructor throws', async () => { + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => { + throw new Error('constructor error'); + }); + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const onChunk = sinon.stub(); + + await forEachChunkedJsonStore( + '/nonexistent/path', + 'index.json', + { chunkReadLogLabel: 'test', onOpenError, onEmptyIndexer }, + onChunk, + ); + + expect(onOpenError.callCount).to.equal(1); + expect(onEmptyIndexer.callCount).to.equal(0); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onEmptyIndexer when the index file exists but has no entries', async () => { + const tmpDir = path.join(os.tmpdir(), `cjr-empty-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + fsReal.writeFileSync(path.join(tmpDir, 'index.json'), '{}'); + + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const onChunk = sinon.stub(); + + await forEachChunkedJsonStore( + tmpDir, + 'index.json', + { chunkReadLogLabel: 'test', onOpenError, onEmptyIndexer }, + onChunk, + ); + + expect(onEmptyIndexer.callCount).to.equal(1); + expect(onChunk.callCount).to.equal(0); + }); + + it('calls onChunk with records when the index has entries', async () => { + const tmpDir = path.join(os.tmpdir(), `cjr-chunks-${Date.now()}`); + fsReal.mkdirSync(tmpDir, { recursive: true }); + fsReal.writeFileSync(path.join(tmpDir, 'index.json'), '{"0": true}'); + + const record = { uid: 'field-1', name: 'My Field' }; + sinon.stub(FsUtility.prototype, 'indexFileContent' as any).get(() => ({ '0': true })); + sinon.stub(FsUtility.prototype, 'readChunkFiles' as any).get(() => ({ + next: sinon.stub().resolves({ 'field-1': record }), + })); + + const onOpenError = sinon.stub(); + const onEmptyIndexer = sinon.stub(); + const collected: unknown[] = []; + + await forEachChunkedJsonStore( + tmpDir, + 'index.json', + { chunkReadLogLabel: 'fields', onOpenError, onEmptyIndexer }, + async (records) => { collected.push(...records); }, + ); + + expect(onOpenError.callCount).to.equal(0); + expect(onEmptyIndexer.callCount).to.equal(0); + expect(collected).to.deep.equal([record]); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/concurrent-batch.test.ts b/packages/contentstack-asset-management/test/unit/utils/concurrent-batch.test.ts new file mode 100644 index 000000000..cc95ad49e --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/concurrent-batch.test.ts @@ -0,0 +1,54 @@ +import { expect } from 'chai'; + +import { chunkArray, runInBatches } from '../../../src/utils/concurrent-batch'; + +describe('concurrent-batch', () => { + describe('chunkArray', () => { + it('should split an array into chunks of at most `size`', () => { + expect(chunkArray([1, 2, 3, 4, 5], 2)).to.deep.equal([[1, 2], [3, 4], [5]]); + }); + + it('should return a single chunk when size >= length', () => { + expect(chunkArray([1, 2, 3], 10)).to.deep.equal([[1, 2, 3]]); + }); + + it('should return the whole array as one chunk when size <= 0', () => { + expect(chunkArray([1, 2, 3], 0)).to.deep.equal([[1, 2, 3]]); + }); + + it('should return [] for an empty array', () => { + expect(chunkArray([], 3)).to.deep.equal([]); + }); + }); + + describe('runInBatches', () => { + it('should invoke fn for every item with the correct absolute index', async () => { + const seen: Array<{ item: string; index: number }> = []; + await runInBatches(['a', 'b', 'c'], 2, async (item, index) => { + seen.push({ item, index }); + }); + expect(seen.sort((a, b) => a.index - b.index)).to.deep.equal([ + { item: 'a', index: 0 }, + { item: 'b', index: 1 }, + { item: 'c', index: 2 }, + ]); + }); + + it('should not abort the batch when one task rejects (fault-tolerant)', async () => { + const completed: number[] = []; + await runInBatches([1, 2, 3, 4], 2, async (n) => { + if (n === 2) throw new Error('boom'); + completed.push(n); + }); + expect(completed.sort((a, b) => a - b)).to.deep.equal([1, 3, 4]); + }); + + it('should be a no-op for an empty array', async () => { + let called = false; + await runInBatches([], 5, async () => { + called = true; + }); + expect(called).to.equal(false); + }); + }); +}); diff --git a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts index 20f122d82..d9b86a60a 100644 --- a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts +++ b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts @@ -141,14 +141,27 @@ describe('CSAssetsAdapter', () => { }); describe('getWorkspaceFields', () => { - it('should GET /api/fields and return the response data', async () => { + it('should GET /api/fields (paginated) and return the merged fields', async () => { const fieldsResponse = { count: 1, relation: 'org', fields: [{ uid: 'f1' }] }; getStub.resolves({ status: 200, data: fieldsResponse }); const adapter = new CSAssetsAdapter(baseConfig); const result = await adapter.getWorkspaceFields('sp-1'); - expect(getStub.firstCall.args[0]).to.equal('/api/fields'); - expect(result).to.deep.equal(fieldsResponse); + expect(getStub.firstCall.args[0]).to.include('/api/fields'); + expect(getStub.firstCall.args[0]).to.include('skip=0'); + expect(result).to.deep.equal({ fields: [{ uid: 'f1' }], count: 1 }); + }); + + it('should fetch all fields across multiple pages', async () => { + getStub.onCall(0).resolves({ status: 200, data: { fields: [{ uid: 'f1' }, { uid: 'f2' }], count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f3' }], count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceFields('sp-1', 2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.count).to.equal(3); + expect(result.fields).to.have.lengthOf(3); }); }); @@ -182,16 +195,30 @@ describe('CSAssetsAdapter', () => { }); describe('getWorkspaceAssetTypes', () => { - it('should GET /api/asset_types?include_fields=true and return the response data', async () => { + it('should GET /api/asset_types?include_fields=true (paginated) and return the merged asset types', async () => { const atResponse = { count: 1, relation: 'org', asset_types: [{ uid: 'at1' }] }; getStub.resolves({ status: 200, data: atResponse }); const adapter = new CSAssetsAdapter(baseConfig); - const result = await adapter.getWorkspaceAssetTypes('sp-1'); + const result: unknown = await adapter.getWorkspaceAssetTypes('sp-1'); const path = getStub.firstCall.args[0] as string; expect(path).to.include('/api/asset_types'); expect(path).to.include('include_fields=true'); - expect(result).to.deep.equal(atResponse); + expect(path).to.include('skip=0'); + expect(result).to.deep.equal({ asset_types: [{ uid: 'at1' }], count: 1 }); + }); + + it('should fetch all asset types across multiple pages, preserving include_fields', async () => { + getStub.onCall(0).resolves({ status: 200, data: { asset_types: [{ uid: 'at1' }, { uid: 'at2' }], count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { asset_types: [{ uid: 'at3' }], count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceAssetTypes('sp-1', 2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('include_fields=true'); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.count).to.equal(3); + expect(result.asset_types).to.have.lengthOf(3); }); }); @@ -206,14 +233,435 @@ describe('CSAssetsAdapter', () => { expect(path).to.include('addl_fields=users'); }); - it('should return empty string and no "?" when params are empty', async () => { + it('should append pagination params (limit/skip) for paginated fields collection', async () => { getStub.resolves({ status: 200, data: { count: 0, relation: '', fields: [] } }); const adapter = new CSAssetsAdapter(baseConfig); await adapter.getWorkspaceFields('sp-1'); const path = getStub.firstCall.args[0] as string; - expect(path).to.equal('/api/fields'); - expect(path).to.not.include('?'); + expect(path).to.include('/api/fields?'); + expect(path).to.include('limit='); + expect(path).to.include('skip=0'); + }); + }); + + describe('listSpaces (paginated)', () => { + it('should return all spaces in a single page when count <= pageSize', async () => { + const spaces = [{ uid: 'sp-1' }, { uid: 'sp-2' }]; + getStub.resolves({ status: 200, data: { spaces, count: 2 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(100, 5); + + expect(getStub.callCount).to.equal(1); + expect(getStub.firstCall.args[0]).to.include('/api/spaces'); + expect(getStub.firstCall.args[0]).to.include('limit=100'); + expect(getStub.firstCall.args[0]).to.include('skip=0'); + expect(result.spaces).to.deep.equal(spaces); + expect(result.count).to.equal(2); + }); + + it('should issue additional page requests when total exceeds first page', async () => { + const page1 = Array.from({ length: 2 }, (_, i) => ({ uid: `sp-${i}` })); + const page2 = Array.from({ length: 1 }, (_, i) => ({ uid: `sp-${i + 2}` })); + getStub.onCall(0).resolves({ status: 200, data: { spaces: page1, count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { spaces: page2, count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(2, 5); + + expect(getStub.callCount).to.equal(2); + expect(getStub.secondCall.args[0]).to.include('skip=2'); + expect(result.spaces).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + + it('should return empty spaces when count is 0', async () => { + getStub.resolves({ status: 200, data: { spaces: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(); + + expect(getStub.callCount).to.equal(1); + expect(result.spaces).to.deep.equal([]); + expect(result.count).to.equal(0); + }); + + it('should batch additional page requests by fetchConcurrency', async () => { + // 5 total, pageSize=1, concurrency=2 → 4 additional pages in 2 batches + const pages = Array.from({ length: 5 }, (_, i) => [{ uid: `sp-${i}` }]); + getStub.onCall(0).resolves({ status: 200, data: { spaces: pages[0], count: 5 } }); + getStub.onCall(1).resolves({ status: 200, data: { spaces: pages[1], count: 5 } }); + getStub.onCall(2).resolves({ status: 200, data: { spaces: pages[2], count: 5 } }); + getStub.onCall(3).resolves({ status: 200, data: { spaces: pages[3], count: 5 } }); + getStub.onCall(4).resolves({ status: 200, data: { spaces: pages[4], count: 5 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(1, 2); + + expect(getStub.callCount).to.equal(5); + expect(result.spaces).to.have.lengthOf(5); + }); + + it('should request each page with its OWN skip when pages run concurrently (no shared-mutation race)', async () => { + // total 6, pageSize 2, concurrency 5 → page0 inline, then skips [2,4] in a single concurrent batch. + // If makeAPICall did not snapshot queryParam, both concurrent calls would read the last skip. + const bySkip: Record = { + '0': { spaces: [{ uid: 's0' }, { uid: 's1' }], count: 6 }, + '2': { spaces: [{ uid: 's2' }, { uid: 's3' }], count: 6 }, + '4': { spaces: [{ uid: 's4' }, { uid: 's5' }], count: 6 }, + }; + getStub.callsFake(async (path: string) => { + const skip = new URL(`https://x${path}`).searchParams.get('skip') ?? '0'; + return { status: 200, data: bySkip[skip] }; + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.listSpaces(2, 5); + + const uids = (result.spaces as Array<{ uid: string }>).map((s) => s.uid).sort(); + expect(uids).to.deep.equal(['s0', 's1', 's2', 's3', 's4', 's5']); + const requestedSkips = getStub.getCalls().map((c) => new URL(`https://x${c.args[0]}`).searchParams.get('skip')); + expect([...requestedSkips].sort()).to.deep.equal(['0', '2', '4']); + }); + }); + + describe('makeConcurrentCall', () => { + it('should be a no-op for empty apiBatches and never hang', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.makeConcurrentCall({ module: 'noop', apiBatches: [] }); + expect(getStub.called).to.equal(false); + }); + + it('should invoke the promisifyHandler once per element with correct index/batchIndex', async () => { + const adapter = new CSAssetsAdapter(baseConfig); + const seen: Array<{ index: number; batchIndex: number; element: unknown; isLastRequest: boolean }> = []; + await adapter.makeConcurrentCall({ module: 'downloads', apiBatches: [['a', 'b'], ['c']] }, async (input) => { + seen.push({ + index: input.index, + batchIndex: input.batchIndex, + element: input.element, + isLastRequest: input.isLastRequest, + }); + }); + expect(seen).to.deep.equal([ + { index: 0, batchIndex: 0, element: 'a', isLastRequest: false }, + { index: 1, batchIndex: 0, element: 'b', isLastRequest: false }, + { index: 0, batchIndex: 1, element: 'c', isLastRequest: true }, + ]); + }); + }); + + describe('retry on transient failures', () => { + // retryBaseDelayMs: 0 → instant retries (no wall-clock backoff in tests). + const retryConfig: CSAssetsAPIConfig = { ...baseConfig, retryBaseDelayMs: 0 }; + + it('should retry a 429 and then succeed', async () => { + getStub.onCall(0).resolves({ status: 429, data: {} }); + getStub.onCall(1).resolves({ status: 429, data: {} }); + getStub.onCall(2).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); + const adapter = new CSAssetsAdapter(retryConfig); + const result = await adapter.getWorkspaceFields('sp-1'); + + expect(getStub.callCount).to.equal(3); + expect(result.fields).to.deep.equal([{ uid: 'f1' }]); + }); + + it('should retry a 5xx and then succeed', async () => { + getStub.onCall(0).resolves({ status: 503, data: {} }); + getStub.onCall(1).resolves({ status: 200, data: { fields: [{ uid: 'f1' }], count: 1 } }); + const adapter = new CSAssetsAdapter(retryConfig); + const result = await adapter.getWorkspaceFields('sp-1'); + + expect(getStub.callCount).to.equal(2); + expect(result.fields).to.deep.equal([{ uid: 'f1' }]); + }); + + it('should NOT retry a 404 (terminal) and surface a normalized error', async () => { + getStub.resolves({ status: 404, data: { error: 'not found' } }); + const adapter = new CSAssetsAdapter(retryConfig); + let error: unknown; + try { + await adapter.getWorkspaceFields('sp-1'); + } catch (e) { + error = e; + } + expect(getStub.callCount).to.equal(1); + expect((error as Error)?.message).to.include('CS Assets API GET failed'); + }); + }); + + describe('getWorkspaceAssets (paginated)', () => { + it('should fetch all assets across multiple pages', async () => { + const page1 = [{ uid: 'a-1' }, { uid: 'a-2' }]; + const page2 = [{ uid: 'a-3' }]; + getStub.onCall(0).resolves({ status: 200, data: { assets: page1, count: 3 } }); + getStub.onCall(1).resolves({ status: 200, data: { assets: page2, count: 3 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceAssets('sp-1', undefined, 2, 5) as any; + + expect(result.assets).to.have.lengthOf(3); + expect(result.count).to.equal(3); + }); + + it('should include workspace query param when workspaceUid is provided', async () => { + getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1', 'ws-main', 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('workspace=ws-main'); + }); + + it('should NOT include workspace param when workspaceUid is undefined', async () => { + getStub.resolves({ status: 200, data: { assets: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceAssets('sp-1', undefined, 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.not.include('workspace='); + }); + }); + + describe('getWorkspaceFolders (paginated)', () => { + it('should fetch all folders across multiple pages', async () => { + const page1 = [{ uid: 'f-1' }]; + const page2 = [{ uid: 'f-2' }]; + getStub.onCall(0).resolves({ status: 200, data: { folders: page1, count: 2 } }); + getStub.onCall(1).resolves({ status: 200, data: { folders: page2, count: 2 } }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.getWorkspaceFolders('sp-1', undefined, 1, 5) as any; + + expect(result.folders).to.have.lengthOf(2); + expect(result.count).to.equal(2); + }); + + it('should include workspace param when workspaceUid is provided', async () => { + getStub.resolves({ status: 200, data: { folders: [], count: 0 } }); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.getWorkspaceFolders('sp-1', 'ws-main', 100, 5); + + const path = getStub.firstCall.args[0] as string; + expect(path).to.include('workspace=ws-main'); + }); + }); + + describe('POST methods (createSpace, createFolder, createField, createAssetType, bulkDelete, bulkMove)', () => { + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + fetchStub = sinon.stub(global, 'fetch' as any); + }); + + const okJsonResponse = (data: unknown) => ({ + ok: true, + json: async () => data, + text: async () => JSON.stringify(data), + }); + + const failResponse = (status: number, body = 'error body') => ({ + ok: false, + status, + json: async () => ({}), + text: async () => body, + }); + + describe('createSpace', () => { + it('POSTs to /api/spaces and returns the created space', async () => { + const created = { space: { uid: 'new-space-uid', title: 'My Space' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createSpace({ title: 'My Space' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces'); + expect(opts.method).to.equal('POST'); + expect(result).to.deep.equal(created); + }); + + it('throws when POST returns non-ok status', async () => { + fetchStub.resolves(failResponse(400, 'bad request')); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.createSpace({ title: 'Bad Space' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('400'); + } + }); + }); + + describe('createFolder', () => { + it('POSTs to /api/spaces/{spaceUid}/folders with space_key header', async () => { + const created = { folder: { uid: 'folder-new' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createFolder('sp-1', { title: 'Docs' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/folders'); + expect(opts.headers['space_key']).to.equal('sp-1'); + expect(result).to.deep.equal(created); + }); + + it('URL-encodes spaceUid with special characters', async () => { + fetchStub.resolves(okJsonResponse({ folder: { uid: 'f1' } })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.createFolder('sp uid/1', { title: 'X' }); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('sp%20uid%2F1'); + }); + }); + + describe('createField', () => { + it('POSTs to /api/fields and returns the created field', async () => { + const created = { field: { uid: 'field-1' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createField({ uid: 'field-1', label: 'My Field' } as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('/api/fields'); + expect(result).to.deep.equal(created); + }); + }); + + describe('createAssetType', () => { + it('POSTs to /api/asset_types and returns the created asset type', async () => { + const created = { asset_type: { uid: 'at-1' } }; + fetchStub.resolves(okJsonResponse(created)); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.createAssetType({ uid: 'at-1' } as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('/api/asset_types'); + expect(result).to.deep.equal(created); + }); + }); + + describe('bulkDeleteAssets', () => { + it('POSTs to the bulk delete endpoint with workspace query param', async () => { + fetchStub.resolves(okJsonResponse({ deleted: 2 })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkDeleteAssets('sp-1', 'ws-main', { asset_uids: ['a1', 'a2'] } as any); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets/bulk/delete'); + expect(url).to.include('workspace=ws-main'); + expect(opts.headers['space_key']).to.equal('sp-1'); + }); + + it('uses "main" as default workspace uid', async () => { + fetchStub.resolves(okJsonResponse({})); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkDeleteAssets('sp-1', undefined as any, {} as any); + + const [url] = fetchStub.firstCall.args; + expect(url).to.include('workspace=main'); + }); + }); + + describe('bulkMoveAssets', () => { + it('POSTs to the bulk-move endpoint with workspace query param', async () => { + fetchStub.resolves(okJsonResponse({ moved: 1 })); + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], folder_uid: 'f1' } as any); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets/bulk-move'); + expect(url).to.include('workspace=ws-main'); + expect(opts.headers['space_key']).to.equal('sp-1'); + }); + }); + + describe('postJson error handling', () => { + it('wraps non-API errors in a consistent error message', async () => { + fetchStub.rejects(new Error('network failure')); + const adapter = new CSAssetsAdapter(baseConfig); + + try { + await adapter.createField({} as any); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API POST failed'); + expect(err.message).to.include('network failure'); + } + }); + }); + + describe('uploadAsset', () => { + const os = require('os'); + const path = require('path'); + const fsReal = require('fs'); + + it('reads the file, builds multipart form, and POSTs to /api/spaces/{uid}/assets', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-test-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'fake-image-content'); + fetchStub.resolves(okJsonResponse({ asset: { uid: 'new-asset', url: 'https://cdn.com/x.png' } })); + + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.uploadAsset('sp-1', tmpFile, { title: 'My Image' }); + + const [url, opts] = fetchStub.firstCall.args; + expect(url).to.include('/api/spaces/sp-1/assets'); + expect(opts.method).to.equal('POST'); + expect(opts.headers['space_key']).to.equal('sp-1'); + expect(result).to.deep.equal({ asset: { uid: 'new-asset', url: 'https://cdn.com/x.png' } }); + + fsReal.unlinkSync(tmpFile); + }); + + it('appends description and parent_uid to the form when provided', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-test-desc-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + const formAppendSpy = sinon.spy(FormData.prototype, 'append'); + fetchStub.resolves(okJsonResponse({ asset: { uid: 'a1', url: 'https://cdn.com/a1.png' } })); + + const adapter = new CSAssetsAdapter(baseConfig); + await adapter.uploadAsset('sp-1', tmpFile, { + title: 'T', description: 'Desc', parent_uid: 'folder-uid', + }); + + const appendCalls = formAppendSpy.getCalls().map((c) => c.args[0]); + expect(appendCalls).to.include('description'); + expect(appendCalls).to.include('parent_uid'); + + fsReal.unlinkSync(tmpFile); + }); + + it('throws when multipart POST returns non-ok status', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-fail-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + fetchStub.resolves(failResponse(413, 'file too large')); + + const adapter = new CSAssetsAdapter(baseConfig); + try { + await adapter.uploadAsset('sp-1', tmpFile, { title: 'Big File' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('413'); + } + + fsReal.unlinkSync(tmpFile); + }); + + it('wraps network errors from multipart fetch in a consistent error message', async () => { + const tmpFile = path.join(os.tmpdir(), `upload-net-${Date.now()}.png`); + fsReal.writeFileSync(tmpFile, 'data'); + fetchStub.rejects(new Error('connection reset')); + + const adapter = new CSAssetsAdapter(baseConfig); + try { + await adapter.uploadAsset('sp-1', tmpFile, { title: 'File' }); + expect.fail('should have thrown'); + } catch (err: any) { + expect(err.message).to.include('CS Assets API multipart POST failed'); + expect(err.message).to.include('connection reset'); + } + + fsReal.unlinkSync(tmpFile); + }); }); }); }); diff --git a/packages/contentstack-asset-management/test/unit/utils/retry.test.ts b/packages/contentstack-asset-management/test/unit/utils/retry.test.ts new file mode 100644 index 000000000..c3bd30a6c --- /dev/null +++ b/packages/contentstack-asset-management/test/unit/utils/retry.test.ts @@ -0,0 +1,98 @@ +import { expect } from 'chai'; + +import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from '../../../src/utils/retry'; + +describe('retry', () => { + describe('isRetryableStatus', () => { + it('treats 429 and 5xx as retryable, others not', () => { + expect(isRetryableStatus(429)).to.equal(true); + expect(isRetryableStatus(500)).to.equal(true); + expect(isRetryableStatus(503)).to.equal(true); + expect(isRetryableStatus(404)).to.equal(false); + expect(isRetryableStatus(400)).to.equal(false); + expect(isRetryableStatus(200)).to.equal(false); + }); + }); + + describe('parseRetryAfterMs', () => { + it('parses delta-seconds into ms', () => { + expect(parseRetryAfterMs('2')).to.equal(2000); + }); + + it('returns undefined for null/empty', () => { + expect(parseRetryAfterMs(null)).to.equal(undefined); + expect(parseRetryAfterMs('')).to.equal(undefined); + }); + + it('parses an HTTP date into a non-negative ms delay', () => { + const value = parseRetryAfterMs(new Date(Date.now() + 1000).toUTCString()); + expect(value).to.be.a('number'); + expect(value as number).to.be.at.least(0); + }); + }); + + describe('withRetry', () => { + it('returns immediately on success without retrying', async () => { + let calls = 0; + const result = await withRetry( + async () => { + calls += 1; + return 'ok'; + }, + { baseDelayMs: 0 }, + ); + expect(result).to.equal('ok'); + expect(calls).to.equal(1); + }); + + it('retries a RetryableHttpError up to `retries` times then rethrows', async () => { + let calls = 0; + let error: unknown; + try { + await withRetry( + async () => { + calls += 1; + throw new RetryableHttpError('boom', 503); + }, + { retries: 2, baseDelayMs: 0 }, + ); + } catch (e) { + error = e; + } + expect(calls).to.equal(3); // initial attempt + 2 retries + expect(error).to.be.instanceOf(RetryableHttpError); + }); + + it('succeeds after transient failures', async () => { + let calls = 0; + const result = await withRetry( + async () => { + calls += 1; + if (calls < 3) throw new RetryableHttpError('transient', 500); + return calls; + }, + { retries: 5, baseDelayMs: 0 }, + ); + expect(result).to.equal(3); + expect(calls).to.equal(3); + }); + + it('does NOT retry a non-RetryableHttpError (terminal)', async () => { + let calls = 0; + let error: unknown; + try { + await withRetry( + async () => { + calls += 1; + throw new Error('terminal'); + }, + { retries: 3, baseDelayMs: 0 }, + ); + } catch (e) { + error = e; + } + expect(calls).to.equal(1); + expect((error as Error).message).to.equal('terminal'); + }); + }); +}); diff --git a/packages/contentstack-audit/.eslintignore b/packages/contentstack-audit/.eslintignore deleted file mode 100644 index 9b1c8b133..000000000 --- a/packages/contentstack-audit/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -/dist diff --git a/packages/contentstack-audit/.eslintrc b/packages/contentstack-audit/.eslintrc deleted file mode 100644 index 7b846193c..000000000 --- a/packages/contentstack-audit/.eslintrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "extends": [ - "oclif", - "oclif-typescript" - ] -} diff --git a/packages/contentstack-audit/eslint.config.js b/packages/contentstack-audit/eslint.config.js new file mode 100644 index 000000000..9b1d96ae5 --- /dev/null +++ b/packages/contentstack-audit/eslint.config.js @@ -0,0 +1,12 @@ +import oclif from 'eslint-config-oclif'; +import oclifTypescript from 'eslint-config-oclif-typescript'; + +export default [ + oclif, + oclifTypescript, + { + ignores: [ + 'dist/**/*', + ], + }, +]; diff --git a/packages/contentstack-audit/package.json b/packages/contentstack-audit/package.json index 7da2cc83c..12cf48a73 100644 --- a/packages/contentstack-audit/package.json +++ b/packages/contentstack-audit/package.json @@ -1,6 +1,6 @@ { "name": "@contentstack/cli-audit", - "version": "2.0.0-beta.13", + "version": "2.0.0-beta.14", "description": "Contentstack audit plugin", "author": "Contentstack CLI", "homepage": "https://github.com/contentstack/cli", @@ -18,8 +18,8 @@ "/oclif.manifest.json" ], "dependencies": { - "@contentstack/cli-command": "~2.0.0-beta.8", - "@contentstack/cli-utilities": "~2.0.0-beta.9", + "@contentstack/cli-command": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", "@oclif/core": "^4.11.4", "chalk": "^5.6.2", "fast-csv": "^4.3.6", @@ -34,7 +34,7 @@ "@types/mocha": "^10.0.10", "@types/node": "^20.17.50", "chai": "^4.5.0", - "eslint": "^9.26.0", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.62", "eslint-config-oclif-typescript": "^3.1.14", "mocha": "^10.8.2", @@ -59,7 +59,7 @@ }, "scripts": { "build": "pnpm compile && oclif manifest && oclif readme", - "lint": "eslint . --ext .ts --config .eslintrc", + "lint": "eslint .", "postpack": "shx rm -f oclif.manifest.json", "posttest": "npm run lint", "compile": "tsc -b tsconfig.json", @@ -71,7 +71,7 @@ "test:unit": "mocha --timeout 10000 --forbid-only --file test/unit/logger-config.js \"test/unit/**/*.test.ts\"" }, "engines": { - "node": ">=16" + "node": ">=22.0.0" }, "bugs": "https://github.com/contentstack/cli/issues", "keywords": [ diff --git a/packages/contentstack-bootstrap/.eslintrc b/packages/contentstack-bootstrap/.eslintrc deleted file mode 100644 index 7b1ccd7f1..000000000 --- a/packages/contentstack-bootstrap/.eslintrc +++ /dev/null @@ -1,41 +0,0 @@ -{ - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended", - "plugin:mocha/recommended" - ], - "parser": "@typescript-eslint/parser", - "plugins": [ - "@typescript-eslint", - "mocha" - ], - "rules": { - "unicorn/no-abusive-eslint-disable": "off", - "@typescript-eslint/no-use-before-define": "off", - "@typescript-eslint/ban-ts-ignore": "off", - "indent": "off", - "object-curly-spacing": "off", - "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], - "mocha/no-async-describe": "off", - "mocha/no-identical-title": "off", - "mocha/no-mocha-arrows": "off", - "mocha/no-setup-in-describe": "off", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-var-requires": "off", - "prefer-const": "error", - "no-fallthrough": "error", - "no-prototype-builtins": "off" - }, - "env": { - "node": true, - "mocha": true - }, - "overrides": [ - { - "files": ["*.d.ts"], - "rules": { - "@typescript-eslint/no-explicit-any": "off" - } - } - ] -} \ No newline at end of file diff --git a/packages/contentstack-bootstrap/eslint.config.js b/packages/contentstack-bootstrap/eslint.config.js new file mode 100644 index 000000000..708e85dd4 --- /dev/null +++ b/packages/contentstack-bootstrap/eslint.config.js @@ -0,0 +1,53 @@ +import tseslint from 'typescript-eslint'; +import globals from 'globals'; +import mocha from 'eslint-plugin-mocha'; + +export default [ + ...tseslint.configs.recommended, + + { + languageOptions: { + parser: tseslint.parser, + globals: { + ...globals.node, + ...globals.mocha, + }, + }, + + plugins: { + '@typescript-eslint': tseslint.plugin, + mocha: mocha, + }, + + rules: { + 'unicorn/no-abusive-eslint-disable': 'off', + '@typescript-eslint/no-use-before-define': 'off', + '@typescript-eslint/ban-ts-ignore': 'off', + indent: 'off', + 'object-curly-spacing': 'off', + '@typescript-eslint/no-unused-vars': [ + 'error', + { + argsIgnorePattern: '^_', + }, + ], + 'mocha/no-async-describe': 'off', + 'mocha/no-identical-title': 'off', + 'mocha/no-mocha-arrows': 'off', + 'mocha/no-setup-in-describe': 'off', + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-var-requires': 'off', + 'prefer-const': 'error', + 'no-fallthrough': 'error', + 'no-prototype-builtins': 'off', + }, + }, + + { + files: ['*.d.ts'], + + rules: { + '@typescript-eslint/no-explicit-any': 'off', + }, + }, +]; diff --git a/packages/contentstack-bootstrap/package.json b/packages/contentstack-bootstrap/package.json index bf13ad528..c7f0a45e5 100644 --- a/packages/contentstack-bootstrap/package.json +++ b/packages/contentstack-bootstrap/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-bootstrap", "description": "Bootstrap contentstack apps", - "version": "2.0.0-beta.20", + "version": "2.0.0-beta.21", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "scripts": { @@ -16,14 +16,14 @@ "test:report": "nyc --reporter=lcov mocha \"test/**/*.test.js\"" }, "dependencies": { - "@contentstack/cli-cm-seed": "~2.0.0-beta.20", - "@contentstack/cli-command": "~2.0.0-beta.8", - "@contentstack/cli-utilities": "~2.0.0-beta.9", - "@contentstack/cli-config": "~2.0.0-beta.11", + "@contentstack/cli-cm-seed": "~2.0.0-beta.21", + "@contentstack/cli-command": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", + "@contentstack/cli-config": "~2.0.0-beta.12", "@oclif/core": "^4.11.4", "inquirer": "12.11.1", "mkdirp": "^2.1.6", - "tar": "^7.5.15" + "tar": "^7.5.16" }, "devDependencies": { "@oclif/test": "^4.1.18", @@ -32,7 +32,7 @@ "@types/node": "^18.11.9", "@types/tar": "^6.1.13", "chai": "^4.5.0", - "eslint": "^9.26.0", + "eslint": "^10.5.0", "mocha": "10.8.2", "nyc": "^15.1.0", "oclif": "^4.23.8", @@ -41,7 +41,7 @@ "typescript": "^5.9.3" }, "engines": { - "node": ">=14.0.0" + "node": ">=22.0.0" }, "files": [ "/lib", diff --git a/packages/contentstack-branches/.eslintignore b/packages/contentstack-branches/.eslintignore deleted file mode 100644 index 7f0bb8ff2..000000000 --- a/packages/contentstack-branches/.eslintignore +++ /dev/null @@ -1,4 +0,0 @@ -# Build files -/lib -/test -/types \ No newline at end of file diff --git a/packages/contentstack-branches/.eslintrc b/packages/contentstack-branches/.eslintrc deleted file mode 100644 index cb46553b0..000000000 --- a/packages/contentstack-branches/.eslintrc +++ /dev/null @@ -1,55 +0,0 @@ -{ - "env": { - "node": true - }, - "parser": "@typescript-eslint/parser", - "parserOptions": { - "project": "tsconfig.json", - "sourceType": "module" - }, - "extends": [ - // "oclif", - "oclif-typescript", - "plugin:@typescript-eslint/recommended" - ], - "rules": { - "@typescript-eslint/no-unused-vars": [ - "error", - { - "args": "none" - } - ], - "@typescript-eslint/prefer-namespace-keyword": "error", - "@typescript-eslint/quotes": [ - "error", - "single", - { - "avoidEscape": true, - "allowTemplateLiterals": true - } - ], - "semi": "off", - "@typescript-eslint/type-annotation-spacing": "error", - "@typescript-eslint/no-redeclare": "off", - "eqeqeq": [ - "error", - "smart" - ], - "id-match": "error", - "no-eval": "error", - "no-var": "error", - "quotes": "off", - "indent": "off", - "camelcase": "off", - "comma-dangle": "off", - "arrow-parens": "off", - "operator-linebreak": "off", - "object-curly-spacing": "off", - "node/no-missing-import": "off", - "padding-line-between-statements": "off", - "@typescript-eslint/ban-ts-ignore": "off", - "unicorn/no-abusive-eslint-disable": "off", - "unicorn/consistent-function-scoping": "off", - "@typescript-eslint/no-use-before-define": "off" - } -} \ No newline at end of file diff --git a/packages/contentstack-branches/eslint.config.js b/packages/contentstack-branches/eslint.config.js new file mode 100644 index 000000000..fb55b32e6 --- /dev/null +++ b/packages/contentstack-branches/eslint.config.js @@ -0,0 +1,72 @@ +import tseslint from 'typescript-eslint'; +import globals from 'globals'; +import oclif from 'eslint-config-oclif-typescript'; + +export default [ + ...tseslint.configs.recommended, + + oclif, + + { + ignores: [ + 'lib/**/*', + 'test/**/*', + 'types/**/*', + ], + }, + + { + languageOptions: { + parser: tseslint.parser, + parserOptions: { + project: './tsconfig.json', + }, + sourceType: 'module', + globals: { + ...globals.node, + }, + }, + + plugins: { + '@typescript-eslint': tseslint.plugin, + }, + + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { + args: 'none', + }, + ], + '@typescript-eslint/prefer-namespace-keyword': 'error', + '@typescript-eslint/quotes': [ + 'error', + 'single', + { + avoidEscape: true, + allowTemplateLiterals: true, + }, + ], + semi: 'off', + '@typescript-eslint/type-annotation-spacing': 'error', + '@typescript-eslint/no-redeclare': 'off', + eqeqeq: ['error', 'smart'], + 'id-match': 'error', + 'no-eval': 'error', + 'no-var': 'error', + quotes: 'off', + indent: 'off', + camelcase: 'off', + 'comma-dangle': 'off', + 'arrow-parens': 'off', + 'operator-linebreak': 'off', + 'object-curly-spacing': 'off', + 'node/no-missing-import': 'off', + 'padding-line-between-statements': 'off', + '@typescript-eslint/ban-ts-ignore': 'off', + 'unicorn/no-abusive-eslint-disable': 'off', + 'unicorn/consistent-function-scoping': 'off', + '@typescript-eslint/no-use-before-define': 'off', + }, + }, +]; diff --git a/packages/contentstack-branches/package.json b/packages/contentstack-branches/package.json index 07fca6ca4..b6b7f3437 100644 --- a/packages/contentstack-branches/package.json +++ b/packages/contentstack-branches/package.json @@ -1,13 +1,13 @@ { "name": "@contentstack/cli-cm-branches", "description": "Contentstack CLI plugin to do branches operations", - "version": "2.0.0-beta.8", + "version": "2.0.0-beta.9", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { - "@contentstack/cli-command": "~2.0.0-beta.8", + "@contentstack/cli-command": "~2.0.0-beta.9", "@oclif/core": "^4.11.4", - "@contentstack/cli-utilities": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", "chalk": "^5.6.2", "just-diff": "^6.0.2", "lodash": "^4.18.1" @@ -16,7 +16,7 @@ "chai": "^4.5.0", "dotenv": "^16.6.1", "dotenv-expand": "^9.0.0", - "eslint": "^9.26.0", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.62", "mocha": "10.8.2", "nyc": "^15.1.0", @@ -43,7 +43,7 @@ "test:unit:report": "nyc --extension .ts mocha --forbid-only \"test/unit/**/*.test.ts\"" }, "engines": { - "node": ">=14.0.0" + "node": ">=22.0.0" }, "files": [ "/lib", diff --git a/packages/contentstack-bulk-operations/package.json b/packages/contentstack-bulk-operations/package.json index e983c4784..86704d330 100644 --- a/packages/contentstack-bulk-operations/package.json +++ b/packages/contentstack-bulk-operations/package.json @@ -1,6 +1,6 @@ { "name": "@contentstack/cli-bulk-operations", - "version": "2.0.0-beta.1", + "version": "2.0.0-beta.2", "description": "Contentstack CLI plugin for bulk operations", "author": "Contentstack CLI", "homepage": "https://github.com/contentstack/cli-plugins/tree/main/packages/contentstack-bulk-operations", @@ -20,9 +20,9 @@ "/oclif.manifest.json" ], "dependencies": { - "@contentstack/cli-asset-management": "1.0.0-beta.4", - "@contentstack/cli-command": "~2.0.0-beta.8", - "@contentstack/cli-utilities": "~2.0.0-beta.9", + "@contentstack/cli-asset-management": "1.0.0-beta.5", + "@contentstack/cli-command": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", "@contentstack/delivery-sdk": "^5.2.0", "@contentstack/management": "^1.30.2", "lodash": "^4.18.1", @@ -40,7 +40,7 @@ "chai": "^6.2.2", "conventional-changelog-cli": "^5.0.0", "dotenv": "^17.4.2", - "eslint": "^10.3.0", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.162", "eslint-config-oclif-typescript": "^3.1.14", "eslint-config-prettier": "^10.1.8", @@ -94,7 +94,7 @@ "clean": "rm -rf ./lib tsconfig.tsbuildinfo oclif.manifest.json" }, "engines": { - "node": ">=20.19.0" + "node": ">=22.0.0" }, "bugs": "https://github.com/contentstack/cli-plugins/issues", "keywords": [ diff --git a/packages/contentstack-cli-cm-regex-validate/.eslintignore b/packages/contentstack-cli-cm-regex-validate/.eslintignore deleted file mode 100644 index 502167fa0..000000000 --- a/packages/contentstack-cli-cm-regex-validate/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -/lib diff --git a/packages/contentstack-cli-cm-regex-validate/.eslintrc b/packages/contentstack-cli-cm-regex-validate/.eslintrc deleted file mode 100644 index d2b157f5a..000000000 --- a/packages/contentstack-cli-cm-regex-validate/.eslintrc +++ /dev/null @@ -1,23 +0,0 @@ -{ - "extends": ["eslint-config-oclif", "eslint-config-oclif-typescript"], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "rules": { - "unicorn/prefer-module": "off", - "@typescript-eslint/no-require-imports": "off", - "unicorn/no-array-for-each": "off", - "camelcase": "off", - "@typescript-eslint/no-unused-vars": "error", - "quotes": ["error", "single", { "avoidEscape": true }], - "semi": ["error", "never"], - "unicorn/import-style": "off", - "unicorn/prefer-node-protocol": "off", - "unicorn/consistent-function-scoping": "off", - "@typescript-eslint/ban-ts-comment": "off", - "object-curly-spacing": ["error", "never"], - "node/no-missing-import": "off" - } -} diff --git a/packages/contentstack-cli-cm-regex-validate/eslint.config.js b/packages/contentstack-cli-cm-regex-validate/eslint.config.js new file mode 100644 index 000000000..3118238c3 --- /dev/null +++ b/packages/contentstack-cli-cm-regex-validate/eslint.config.js @@ -0,0 +1,36 @@ +import js from '@eslint/js'; +import tseslint from 'typescript-eslint'; +import globals from 'globals'; + +export default [ + { + ignores: ['lib/**/*', 'test/**/*', 'bin/*'], + }, + { + files: ['src/**/*.ts'], + languageOptions: { + parser: tseslint.parser, + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + }, + globals: { + ...globals.node, + }, + }, + plugins: { + '@typescript-eslint': tseslint.plugin, + }, + rules: { + ...js.configs.recommended.rules, + ...tseslint.configs.recommended[1].rules, + '@typescript-eslint/no-require-imports': 'off', + 'camelcase': 'off', + '@typescript-eslint/no-unused-vars': 'error', + 'quotes': ['error', 'single', { avoidEscape: true }], + 'semi': ['error', 'never'], + '@typescript-eslint/ban-ts-comment': 'off', + 'object-curly-spacing': ['error', 'never'], + }, + }, +]; \ No newline at end of file diff --git a/packages/contentstack-cli-cm-regex-validate/package.json b/packages/contentstack-cli-cm-regex-validate/package.json index a0f2fc13b..e51996e1c 100644 --- a/packages/contentstack-cli-cm-regex-validate/package.json +++ b/packages/contentstack-cli-cm-regex-validate/package.json @@ -1,12 +1,12 @@ { "name": "@contentstack/cli-cm-regex-validate", "description": "Validate Fields with Regex Property of Content Type and Global Field in a Stack", - "version": "2.0.0-beta.0", + "version": "2.0.0-beta.1", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli-plugins/issues", "dependencies": { - "@contentstack/cli-command": "~2.0.0-beta.8", - "@contentstack/cli-utilities": "~2.0.0-beta.9", + "@contentstack/cli-command": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", "@contentstack/management": "^1.30.2", "cli-table3": "^0.6.5", "inquirer": "12.11.1", @@ -25,7 +25,7 @@ "@types/node": "^18.19.130", "@types/safe-regex": "^1.1.6", "chai": "^4.5.0", - "eslint": "^9.26.0", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.62", "eslint-config-oclif-typescript": "^3.1.14", "globby": "^11.1.0", @@ -38,7 +38,7 @@ "typescript": "^5.9.3" }, "engines": { - "node": ">=14.0.0" + "node": ">=22.0.0" }, "files": [ "/bin", @@ -74,8 +74,8 @@ "postpack": "rm -f oclif.manifest.json", "test": "jest --detectOpenHandles --silent", "test:unit": "jest --detectOpenHandles --silent", - "posttest": "eslint src/**/*.ts", - "lint": "eslint src/**/*.ts", + "posttest": "eslint src/**/*.ts --fix", + "lint": "eslint src/**/*.ts --fix", "clean": "rm -rf ./lib ./node_modules tsconfig.tsbuildinfo oclif.manifest.json", "version": "oclif readme && git add README.md" }, diff --git a/packages/contentstack-cli-tsgen/.eslintrc.js b/packages/contentstack-cli-tsgen/.eslintrc.js deleted file mode 100644 index 63bf82762..000000000 --- a/packages/contentstack-cli-tsgen/.eslintrc.js +++ /dev/null @@ -1,18 +0,0 @@ -module.exports = { - parser: "@typescript-eslint/parser", - parserOptions: { - ecmaVersion: 2020, - sourceType: "module", - }, - plugins: ["@typescript-eslint"], - extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"], - rules: { - "unicorn/prefer-module": "off", - "unicorn/no-abusive-eslint-disable": "off", - "@typescript-eslint/no-use-before-define": "off", - "node/no-missing-import": "off", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-require-imports": "off", - "no-useless-escape": "off", - }, -}; diff --git a/packages/contentstack-cli-tsgen/eslint.config.js b/packages/contentstack-cli-tsgen/eslint.config.js new file mode 100644 index 000000000..f67ee11b4 --- /dev/null +++ b/packages/contentstack-cli-tsgen/eslint.config.js @@ -0,0 +1,31 @@ +import tseslint from 'typescript-eslint'; + +export default [ + { + ignores: [ + 'lib/**/*', + ], + }, + ...tseslint.configs.recommended, + { + languageOptions: { + parser: tseslint.parser, + parserOptions: { + ecmaVersion: 2020, + sourceType: 'module', + }, + }, + plugins: { + '@typescript-eslint': tseslint.plugin, + }, + rules: { + 'unicorn/prefer-module': 'off', + 'unicorn/no-abusive-eslint-disable': 'off', + '@typescript-eslint/no-use-before-define': 'off', + 'node/no-missing-import': 'off', + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-require-imports': 'off', + 'no-useless-escape': 'off', + }, + }, +]; diff --git a/packages/contentstack-cli-tsgen/package.json b/packages/contentstack-cli-tsgen/package.json index 940a23f99..6f9259696 100644 --- a/packages/contentstack-cli-tsgen/package.json +++ b/packages/contentstack-cli-tsgen/package.json @@ -1,13 +1,13 @@ { "name": "contentstack-cli-tsgen", "description": "Generate TypeScript typings from a Stack.", - "version": "5.0.0-beta.0", + "version": "5.0.0-beta.1", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli-plugins/issues", "dependencies": { - "@contentstack/cli-command": "~2.0.0-beta.8", - "@contentstack/cli-utilities": "~2.0.0-beta.9", - "@contentstack/types-generator": "^3.10.0" + "@contentstack/cli-command": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", + "@contentstack/types-generator": "^3.10.2" }, "devDependencies": { "@oclif/plugin-help": "^6.2.49", @@ -17,7 +17,7 @@ "@typescript-eslint/eslint-plugin": "^8.59.3", "@typescript-eslint/parser": "^8.59.3", "dotenv": "^16.6.1", - "eslint": "^8.57.1", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.165", "eslint-config-oclif-typescript": "^3.1.14", "jest": "^29.7.0", @@ -26,7 +26,7 @@ "typescript": "^5.9.3" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" }, "files": [ "/lib", @@ -58,9 +58,9 @@ "build": "pnpm compile && oclif manifest && oclif readme", "clean": "rm -rf ./lib ./node_modules tsconfig.tsbuildinfo", "compile": "tsc -b tsconfig.json", - "lint": "eslint . --ext .ts --config .eslintrc.js", + "lint": "eslint .", "postpack": "rm -f oclif.manifest.json", - "posttest": "eslint . --ext .ts --config .eslintrc.js --fix", + "posttest": "eslint . --fix", "prepack": "pnpm compile && oclif manifest && oclif readme", "test": "jest --testPathPattern=tests", "test:integration": "jest --testPathPattern=tests/integration", diff --git a/packages/contentstack-clone/.eslintrc b/packages/contentstack-clone/.eslintrc deleted file mode 100644 index 6a9dd0894..000000000 --- a/packages/contentstack-clone/.eslintrc +++ /dev/null @@ -1,54 +0,0 @@ -{ - "env": { - "node": true, - "es2021": true - }, - "parser": "@typescript-eslint/parser", - "parserOptions": { - "project": "tsconfig.json", - "sourceType": "module" - }, - "plugins": [ - "@typescript-eslint" - ], - "extends": [ - "plugin:@typescript-eslint/recommended", - "plugin:@typescript-eslint/recommended-requiring-type-checking" - ], - "ignorePatterns": [ - "lib/**/*", - "test/**/*", - "node_modules/**/*", - "*.js" - ], - "rules": { - "@typescript-eslint/no-unused-vars": [ - "error", - { - "args": "none", - "argsIgnorePattern": "^_", - "varsIgnorePattern": "^_" - } - ], - "@typescript-eslint/prefer-namespace-keyword": "error", - "@typescript-eslint/no-floating-promises": "error", - "@typescript-eslint/no-misused-promises": "error", - "@typescript-eslint/await-thenable": "error", - "quotes": ["error", "single", { "avoidEscape": true, "allowTemplateLiterals": true }], - "semi": "off", - "@typescript-eslint/no-redeclare": "off", - "eqeqeq": ["error", "smart"], - "id-match": "error", - "no-eval": "error", - "no-var": "error", - "@typescript-eslint/no-explicit-any": "warn", - "@typescript-eslint/no-require-imports": "off", - "prefer-const": "error", - "@typescript-eslint/no-unsafe-call": "off", - "@typescript-eslint/no-unsafe-member-access": "off", - "@typescript-eslint/no-unsafe-assignment": "off", - "@typescript-eslint/no-unsafe-return": "off", - "@typescript-eslint/no-unsafe-argument": "off", - "@typescript-eslint/require-await": "off" - } -} diff --git a/packages/contentstack-clone/eslint.config.js b/packages/contentstack-clone/eslint.config.js new file mode 100644 index 000000000..79a076192 --- /dev/null +++ b/packages/contentstack-clone/eslint.config.js @@ -0,0 +1,64 @@ +import tseslint from 'typescript-eslint'; +import globals from 'globals'; + +export default [ + ...tseslint.configs.recommended, + ...tseslint.configs.recommendedTypeChecked, + + { + ignores: [ + 'lib/**/*', + 'test/**/*', + 'node_modules/**/*', + '*.js', + ], + }, + + { + languageOptions: { + parser: tseslint.parser, + parserOptions: { + project: './tsconfig.json', + }, + sourceType: 'module', + globals: { + ...globals.node, + }, + }, + + plugins: { + '@typescript-eslint': tseslint.plugin, + }, + + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { + args: 'none', + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + }, + ], + '@typescript-eslint/prefer-namespace-keyword': 'error', + '@typescript-eslint/no-floating-promises': 'error', + '@typescript-eslint/no-misused-promises': 'error', + '@typescript-eslint/await-thenable': 'error', + quotes: ['error', 'single', { avoidEscape: true, allowTemplateLiterals: true }], + semi: 'off', + '@typescript-eslint/no-redeclare': 'off', + eqeqeq: ['error', 'smart'], + 'id-match': 'error', + 'no-eval': 'error', + 'no-var': 'error', + '@typescript-eslint/no-explicit-any': 'warn', + '@typescript-eslint/no-require-imports': 'off', + 'prefer-const': 'error', + '@typescript-eslint/no-unsafe-call': 'off', + '@typescript-eslint/no-unsafe-member-access': 'off', + '@typescript-eslint/no-unsafe-assignment': 'off', + '@typescript-eslint/no-unsafe-return': 'off', + '@typescript-eslint/no-unsafe-argument': 'off', + '@typescript-eslint/require-await': 'off', + }, + }, +]; diff --git a/packages/contentstack-clone/package.json b/packages/contentstack-clone/package.json index 26ad06779..947ad7c54 100644 --- a/packages/contentstack-clone/package.json +++ b/packages/contentstack-clone/package.json @@ -1,15 +1,15 @@ { "name": "@contentstack/cli-cm-clone", "description": "Contentstack stack clone plugin", - "version": "2.0.0-beta.21", + "version": "2.0.0-beta.22", "author": "Contentstack", "bugs": "https://github.com/rohitmishra209/cli-cm-clone/issues", "dependencies": { "@colors/colors": "^1.6.0", - "@contentstack/cli-cm-export": "~2.0.0-beta.20", - "@contentstack/cli-cm-import": "~2.0.0-beta.20", - "@contentstack/cli-command": "~2.0.0-beta.8", - "@contentstack/cli-utilities": "~2.0.0-beta.9", + "@contentstack/cli-cm-export": "~2.0.0-beta.21", + "@contentstack/cli-cm-import": "~2.0.0-beta.21", + "@contentstack/cli-command": "~2.0.0-beta.9", + "@contentstack/cli-utilities": "~2.0.0-beta.10", "@oclif/core": "^4.11.4", "chalk": "^5.6.2", "inquirer": "12.11.1", @@ -27,7 +27,7 @@ "@types/sinon": "^21.0.0", "@typescript-eslint/eslint-plugin": "^5.62.0", "chai": "^4.5.0", - "eslint": "^9.26.0", + "eslint": "^10.5.0", "eslint-config-oclif": "^6.0.62", "mocha": "^10.8.2", "nyc": "^15.1.0", @@ -37,7 +37,7 @@ "typescript": "^5.9.3" }, "engines": { - "node": ">=14.0.0" + "node": ">=22.0.0" }, "files": [ "/bin", diff --git a/packages/contentstack-content-type/.eslintignore b/packages/contentstack-content-type/.eslintignore deleted file mode 100644 index dc555529d..000000000 --- a/packages/contentstack-content-type/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -/lib \ No newline at end of file diff --git a/packages/contentstack-content-type/.eslintrc b/packages/contentstack-content-type/.eslintrc deleted file mode 100644 index dffeb7eca..000000000 --- a/packages/contentstack-content-type/.eslintrc +++ /dev/null @@ -1,51 +0,0 @@ -{ - "extends": [ - // "oclif", - "oclif-typescript", - "plugin:@typescript-eslint/recommended" - ], - "rules": { - "@typescript-eslint/no-unused-vars": [ - "error", - { - "args": "none" - } - ], - "@typescript-eslint/prefer-namespace-keyword": "error", - "@typescript-eslint/quotes": [ - "error", - "single", - { - "avoidEscape": true, - "allowTemplateLiterals": true - } - ], - "semi": "off", - "@typescript-eslint/type-annotation-spacing": "error", - "@typescript-eslint/no-redeclare": "off", - "eqeqeq": [ - "error", - "smart" - ], - "id-match": "error", - "no-eval": "error", - "no-var": "error", - "quotes": "off", - "indent": "off", - "camelcase": "off", - "comma-dangle": "off", - "arrow-parens": "off", - "operator-linebreak": "off", - "object-curly-spacing": "off", - "node/no-missing-import": "off", - "padding-line-between-statements": "off", - "@typescript-eslint/ban-ts-ignore": "off", - "unicorn/no-abusive-eslint-disable": "off", - "unicorn/consistent-function-scoping": "off", - "@typescript-eslint/no-use-before-define": "off", - "@typescript-eslint/camelcase": "off", - "no-process-exit":"off", - "unicorn/no-process-exit": "off", - "@typescript-eslint/no-var-requires": "off" - } -} \ No newline at end of file diff --git a/packages/contentstack-content-type/README.md b/packages/contentstack-content-type/README.md index ddde8811b..8e5615c93 100644 --- a/packages/contentstack-content-type/README.md +++ b/packages/contentstack-content-type/README.md @@ -54,5 +54,181 @@ $ csdx content-type:details -a "management token" -c "content type" --no-path # Commands +* [`csdx content-type:audit`](#csdx-content-typeaudit) +* [`csdx content-type:compare`](#csdx-content-typecompare) +* [`csdx content-type:compare-remote`](#csdx-content-typecompare-remote) +* [`csdx content-type:details`](#csdx-content-typedetails) +* [`csdx content-type:diagram`](#csdx-content-typediagram) +* [`csdx content-type:list`](#csdx-content-typelist) +## `csdx content-type:audit` + +Display recent changes to a Content Type + +``` +USAGE + $ csdx content-type:audit -c [-s | -a | -a ] [-k | | ] + +FLAGS + -a, --alias= Alias of the management token + -a, --token-alias= Management token alias + -c, --content-type= (required) Content Type UID + -k, --stack-api-key= Stack API Key + -s, --stack= Stack UID + +DESCRIPTION + Display recent changes to a Content Type + +EXAMPLES + $ csdx content-type:audit --stack-api-key "xxxxxxxxxxxxxxxxxxx" --content-type "home_page" + + $ csdx content-type:audit --alias "management token" --content-type "home_page" +``` + +_See code: [src/commands/content-type/audit.ts](https://github.com/contentstack/cli-plugins/blob/main/packages/contentstack-content-type/src/commands/content-type/audit.ts)_ + +## `csdx content-type:compare` + +Compare two Content Type versions + +``` +USAGE + $ csdx content-type:compare -c [-s | -a ] [-k | ] [-a ] [-l -r ] + +FLAGS + -a, --alias= Alias of the management token + -a, --token-alias= Management token alias + -c, --content-type= (required) Content Type UID + -k, --stack-api-key= Stack API Key + -l, --left= Content Type version, i.e. prev version + -r, --right= Content Type version, i.e. later version + -s, --stack= Stack UID + +DESCRIPTION + Compare two Content Type versions + +EXAMPLES + $ csdx content-type:compare --stack-api-key "xxxxxxxxxxxxxxxxxxx" --content-type "home_page" + + $ csdx content-type:compare --stack-api-key "xxxxxxxxxxxxxxxxxxx" --content-type "home_page" --left # --right # + + $ csdx content-type:compare --alias "management token" --content-type "home_page" --left # --right # +``` + +_See code: [src/commands/content-type/compare.ts](https://github.com/contentstack/cli-plugins/blob/main/packages/contentstack-content-type/src/commands/content-type/compare.ts)_ + +## `csdx content-type:compare-remote` + +compare two Content Types on different Stacks + +``` +USAGE + $ csdx content-type:compare-remote (-o -r ) -c + +FLAGS + -c, --content-type= (required) Content Type UID + -o, --origin-stack= (required) Origin Stack API Key + -r, --remote-stack= (required) Remote Stack API Key + +DESCRIPTION + compare two Content Types on different Stacks + +EXAMPLES + $ csdx content-type:compare-remote --origin-stack "xxxxxxxxxxxxxxxxxxx" --remote-stack "xxxxxxxxxxxxxxxxxxx" -content-type "home_page" +``` + +_See code: [src/commands/content-type/compare-remote.ts](https://github.com/contentstack/cli-plugins/blob/main/packages/contentstack-content-type/src/commands/content-type/compare-remote.ts)_ + +## `csdx content-type:details` + +Display Content Type details + +``` +USAGE + $ csdx content-type:details -c [-s | -a ] [-k | ] [-a ] [-p] + +FLAGS + -a, --alias= Alias of the management token + -a, --token-alias= Management token alias + -c, --content-type= (required) Content Type UID + -k, --stack-api-key= Stack API Key + -p, --[no-]path show path column + -s, --stack= Stack UID + +DESCRIPTION + Display Content Type details + +EXAMPLES + $ csdx content-type:details --stack-api-key "xxxxxxxxxxxxxxxxxxx" --content-type "home_page" + + $ csdx content-type:details --alias "management token" --content-type "home_page" + + $ csdx content-type:details --alias "management token" --content-type "home_page" --no-path +``` + +_See code: [src/commands/content-type/details.ts](https://github.com/contentstack/cli-plugins/blob/main/packages/contentstack-content-type/src/commands/content-type/details.ts)_ + +## `csdx content-type:diagram` + +Create a visual diagram of a Stack's Content Types + +``` +USAGE + $ csdx content-type:diagram -o -d portrait|landscape -t svg|dot [-s | -a | -a ] [-k + | | ] + +FLAGS + -a, --alias= Alias of the management token + -a, --token-alias= Management token alias + -d, --direction=