CMG-1101: fix delta reference resolution and locale-migrated tracking

api/src/services/contentful.service.ts

@@ -436,7 +436,7 @@ const processField = (
         return refs;
       }
       const id = lang_value?.sys?.id;
-      if(Array?.isArray(entryId?.id)){
+      if(Array.isArray(entryId?.[id])){
         return entryId?.[id];
       }
       else{

api/src/services/migration.service.ts

@@ -50,7 +50,7 @@ import {
 import { aemService } from './aem.service.js';
 import { requestWithSsoTokenRefresh } from '../utils/sso-request.utils.js';
 import { utilsUpdateCli } from './updateEntryCli.service.js';
-import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js';
+import { clearStaleEntries, enrichConfigWithAssetMapping, enrichConfigWithEntryMapping, enrichConfigWithAssetUpdates, ensureUpdateConfigFile, removeEntriesFromDatabase } from '../utils/entry-update.utils.js';
 import { removeExistingAssets, saveAssetMetadata, AssetUpdate } from '../utils/asset-update.utils.js';
 
 /**
@@ -1262,6 +1262,12 @@ const startMigration = async (req: Request): Promise<any> => {
         iteration,
         safeDeltaMigrationLogPath
       );
+      enrichConfigWithEntryMapping(
+        configFilePath,
+        safePid,
+        iteration,
+        safeDeltaMigrationLogPath
+      );
       enrichConfigWithAssetUpdates(
         configFilePath,
         assetUpdates,

api/src/services/runCli.service.ts

@@ -20,6 +20,7 @@ interface TestStack {
 }
 import { setBasicAuthConfig, setOAuthConfig } from '../utils/config-handler.util.js';
 import writeUidMapping, { writePerLocaleEntryUidMapping } from '../utils/uid-mapper.utils.js';
+import { extractLocalesFromUpdateConfig, recordMigratedLocales } from '../utils/locale-migration.utils.js';
 
 /**
  * Determines log level based on message content without removing ANSI codes
@@ -322,20 +323,54 @@ export const runCli = async (
         ProjectModelLowdb.data.projects[projectIndex].current_step =
           getStepperSteps(ProjectModelLowdb.data.projects[projectIndex]?.iteration).MIGRATION;
         ProjectModelLowdb.data.projects[projectIndex].status = 5;
-        // Record every locale that just successfully migrated so the next delta restart can
-        // tell which locales need a full pass vs delta. Set-union with prior value.
-        const proj: any = ProjectModelLowdb.data.projects[projectIndex];
-        const ranLocales = Array.from(
-          new Set([
-            ...Object.keys(proj?.master_locale ?? {}),
-            ...Object.keys(proj?.locales ?? {}),
-          ]),
-        );
-        const existing: string[] = Array.isArray(proj?.migrated_locales)
-          ? proj.migrated_locales
-          : [];
-        proj.migrated_locales = Array.from(new Set([...existing, ...ranLocales]));
         await ProjectModelLowdb.write();
+
+        // Record every locale that was ACTUALLY processed this run so the next delta
+        // restart can tell which locales still need a full pass vs delta.
+        //
+        // On iteration 1 there's no delta/localize step at all — the whole configured
+        // locale set genuinely gets migrated in one shot, so using the full config is
+        // correct here. From iteration 2 onward, a locale only "ran" this iteration if
+        // it's the master locale (always present) or its entries were actually queued
+        // in this iteration's updated-entries.json (written by removeEntriesFromDatabase
+        // before this CLI import step even started). Using the FULL project locale
+        // config here — instead of what this run actually touched — used to mark
+        // not-yet-migrated locales as done prematurely, permanently skipping them on
+        // every later restart (see CMG delta-migration locale bug).
+        const proj: any = ProjectModelLowdb.data.projects[projectIndex];
+        const currentIteration = proj?.iteration || 1;
+        let ranLocales: string[];
+        if (currentIteration <= 1) {
+          ranLocales = Array.from(
+            new Set([
+              ...Object.keys(proj?.master_locale ?? {}),
+              ...Object.keys(proj?.locales ?? {}),
+            ]),
+          );
+        } else {
+          const updatedEntriesPath = path.join(
+            process.cwd(),
+            DATABASE_FILES.DIRECTORY,
+            projectId,
+            currentIteration.toString(),
+            DATABASE_FILES.UPDATED_ENTRIES,
+          );
+          let updateConfig: Record<string, any> | null = null;
+          if (fs.existsSync(updatedEntriesPath)) {
+            try {
+              updateConfig = JSON.parse(fs.readFileSync(updatedEntriesPath, 'utf-8'));
+            } catch {
+              updateConfig = null;
+            }
+          }
+          ranLocales = Array.from(
+            new Set([
+              ...Object.keys(proj?.master_locale ?? {}),
+              ...extractLocalesFromUpdateConfig(updateConfig),
+            ]),
+          );
+        }
+        await recordMigratedLocales(projectId, ranLocales);
       }
     } else {
       console.info('User not found.');

api/src/utils/entry-update-script.cjs

@@ -4,6 +4,63 @@ const isAssetField = (value) =>
     value && typeof value === 'object' && !Array.isArray(value) &&
     'urlPath' in value && 'filename' in value;
 
+/** Shape produced by processField's 'reference' case: { uid, _content_type_uid }. */
+const isReferenceValue = (value) =>
+    value && typeof value === 'object' && !Array.isArray(value) &&
+    'uid' in value && '_content_type_uid' in value;
+
+const isReferenceArray = (value) =>
+    Array.isArray(value) && value.length > 0 && value.every(isReferenceValue);
+
+/**
+ * Resolves a source-side entry uid to its real Contentstack destination uid.
+ *
+ * The export JSON's reference fields carry the SOURCE cms entry id (see
+ * `contentful.service.ts`'s `createRefrence`), which only happens to equal the
+ * Contentstack uid when entries are imported preserving source ids. The
+ * bulk/master-locale import resolves this correctly via the CLI's own
+ * reference pass; this update path does not, so it needs the same uid-mapper
+ * data the asset resolution above already uses (see `entryMapping`).
+ *
+ * Preference order: per-locale mapping (most precise — handles entries that
+ * ended up as distinct Contentstack uids per locale across iterations) →
+ * flat mapping → identity fallback (keeps existing behavior when no mapping
+ * data exists, e.g. simple setups where source id equals destination uid).
+ */
+const resolveReferenceUid = (sourceUid, locale, entryMapping) => {
+    if (!sourceUid) return sourceUid;
+    const newByLocale = entryMapping?.new?.byLocale?.[locale]?.[sourceUid];
+    if (newByLocale) return newByLocale;
+    const oldByLocale = entryMapping?.old?.byLocale?.[locale]?.[sourceUid];
+    if (oldByLocale) return oldByLocale;
+    const newFlat = entryMapping?.new?.flat?.[sourceUid];
+    if (newFlat) return newFlat;
+    const oldFlat = entryMapping?.old?.flat?.[sourceUid];
+    if (oldFlat) return oldFlat;
+    return sourceUid;
+};
+
+/**
+ * Remaps the uid(s) inside a reference field value (single link object or
+ * array of link objects) to their Contentstack destination uids.
+ */
+const resolveReferenceField = (fieldName, entryUid, value, locale, entryMapping) => {
+    if (isReferenceValue(value)) {
+        const resolved = resolveReferenceUid(value.uid, locale, entryMapping);
+        if (resolved !== value.uid) {
+            console.info(`[${entryUid}] "${fieldName}"${locale ? ` (${locale})` : ''}: resolved reference uid "${value.uid}" → "${resolved}"`);
+        }
+        return { ...value, uid: resolved };
+    }
+    if (isReferenceArray(value)) {
+        return value.map((item) => {
+            const resolved = resolveReferenceUid(item.uid, locale, entryMapping);
+            return { ...item, uid: resolved };
+        });
+    }
+    return value;
+};
+
 /** Export JSON metadata — not Contentstack content-type field UIDs (WordPress entries are flat). */
 const FLAT_PAYLOAD_SKIP = new Set([
     'uid',
@@ -68,7 +125,7 @@ const resolveAssetField = (fieldName, entryUid, updateValue, stackValue, oldMapp
  * WordPress (and similar) write migration JSON with fields at the root (email, url, …).
  * Fetched stack entries keep custom fields under entry.content — merge flat updateData there.
  */
-const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts) => {
+const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping, newMapping, updateOpts, locale, entryMapping) => {
     for (const field of Object.keys(updateData)) {
         if (FLAT_PAYLOAD_SKIP.has(field)) {
             continue;
@@ -89,6 +146,8 @@ const mergeFlatPayloadIntoEntry = async (entry, entryUid, updateData, oldMapping
                 oldMapping,
                 newMapping
             );
+        } else if (isReferenceValue(nextVal) || isReferenceArray(nextVal)) {
+            nextVal = resolveReferenceField(field, entryUid, nextVal, locale, entryMapping);
         }
         entry.content[field] = nextVal;
     }
@@ -103,13 +162,17 @@ module.exports = async ({
     const assetMapping = config.__assetMapping__ || { old: {}, new: {} };
     delete config.__assetMapping__;
 
+    const entryMapping = config.__entryMapping__ || { old: { flat: {}, byLocale: {} }, new: { flat: {}, byLocale: {} } };
+    delete config.__entryMapping__;
+
     // Assets the user chose to update in place (same UID, new file).
     const assetUpdates = Array.isArray(config.__assetUpdates__) ? config.__assetUpdates__ : [];
     delete config.__assetUpdates__;
 
     const oldMapping = assetMapping.old || {};
     const newMapping = assetMapping.new || {};
     console.info(`Asset mappings loaded — old: ${Object.keys(oldMapping).length}, new: ${Object.keys(newMapping).length}`);
+    console.info(`Entry mappings loaded — old: ${Object.keys(entryMapping?.old?.flat || {}).length} flat / ${Object.keys(entryMapping?.old?.byLocale || {}).length} locales, new: ${Object.keys(entryMapping?.new?.flat || {}).length} flat / ${Object.keys(entryMapping?.new?.byLocale || {}).length} locales`);
     console.info(`Asset updates to replace in place: ${assetUpdates.length}`);
 
     const contentTypes = Object.keys(config);
@@ -187,13 +250,21 @@ module.exports = async ({
                                             oldMapping,
                                             newMapping
                                         );
+                                    } else if (isReferenceValue(updateData?.content[field]) || isReferenceArray(updateData?.content[field])) {
+                                        updateData.content[field] = resolveReferenceField(
+                                            field,
+                                            entryUid,
+                                            updateData?.content[field],
+                                            locale,
+                                            entryMapping
+                                        );
                                     }
                                 }
                                 Object.assign(entry?.content, updateData?.content);
                                 await entry.update(updateOpts);
                             } else if (hasStackContent) {
                                 console.info(`[${realEntryUid}] Merging flat migration payload into entry.content (e.g. WordPress export)${locale ? ` for locale "${locale}"` : ''}`);
-                                await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts);
+                                await mergeFlatPayloadIntoEntry(entry, realEntryUid, updateData, oldMapping, newMapping, updateOpts, locale, entryMapping);
                             } else {
                                 if (updateData && entry) {
                                     for (const field of Object.keys(updateData)) {
@@ -206,6 +277,14 @@ module.exports = async ({
                                                 oldMapping,
                                                 newMapping
                                             );
+                                        } else if (isReferenceValue(updateData[field]) || isReferenceArray(updateData[field])) {
+                                            updateData[field] = resolveReferenceField(
+                                                field,
+                                                entryUid,
+                                                updateData[field],
+                                                locale,
+                                                entryMapping
+                                            );
                                         }
                                     }
                                 }
@@ -237,3 +316,7 @@ module.exports = async ({
 module.exports.isAssetField = isAssetField;
 module.exports.resolveAssetField = resolveAssetField;
 module.exports.mergeFlatPayloadIntoEntry = mergeFlatPayloadIntoEntry;
+module.exports.isReferenceValue = isReferenceValue;
+module.exports.isReferenceArray = isReferenceArray;
+module.exports.resolveReferenceUid = resolveReferenceUid;
+module.exports.resolveReferenceField = resolveReferenceField;

api/src/utils/entry-update.utils.ts

@@ -297,6 +297,64 @@ export const enrichConfigWithAssetMapping = (
     writeLogEntry(`Asset references will be resolved using combined old and new mappings`, "enrichConfigWithAssetMapping", loggerPath);
 };
 
+/**
+ * Reads old (previous iteration) and new (current iteration) entry uid mappings
+ * — both the flat source→dest map and the per-locale map written by
+ * `writePerLocaleEntryUidMapping` — and merges them into the updated-entries
+ * config file under `__entryMapping__`.
+ *
+ * This lets the entry-update-script resolve Link(Entry)/reference field values
+ * to their real Contentstack destination uid before writing them onto a
+ * localized (non-master) copy of an entry. Without this, reference fields
+ * written during a locale-add restart keep the export's source-side uid,
+ * which only happens to work when source and destination uids are identical —
+ * the master-locale bulk import resolves this correctly via the Contentstack
+ * CLI's own reference pass, but this update path does not, unless we prime it
+ * with the same uid-mapper data (mirrors `enrichConfigWithAssetMapping`).
+ */
+export const enrichConfigWithEntryMapping = (
+    configFilePath: string,
+    projectId: string,
+    iteration: number,
+    loggerPath?: string
+): void => {
+    const dbBase = path.join(process.cwd(), DATABASE_FILES.DIRECTORY, projectId);
+
+    const readEntryMapper = (iter: number): { flat: Record<string, string>; byLocale: Record<string, Record<string, string>> } => {
+        const p = path.join(dbBase, iter.toString(), DATABASE_FILES.UID_MAPPER);
+        if (!fs.existsSync(p)) return { flat: {}, byLocale: {} };
+        try {
+            const data = JSON.parse(fs.readFileSync(p, "utf-8"));
+            return { flat: data?.entry || {}, byLocale: data?.entryByLocale || {} };
+        } catch (err) {
+            console.error(`Failed to read uid-mapper for iteration ${iter}:`, err);
+            return { flat: {}, byLocale: {} };
+        }
+    };
+
+    const oldEntryMapping = iteration > 1 ? readEntryMapper(iteration - 1) : { flat: {}, byLocale: {} };
+    const newEntryMapping = readEntryMapper(iteration);
+
+    writeLogEntry(
+        `Loaded entry uid mappings — old: ${Object.keys(oldEntryMapping.flat).length} flat / ${Object.keys(oldEntryMapping.byLocale).length} locales, ` +
+        `new: ${Object.keys(newEntryMapping.flat).length} flat / ${Object.keys(newEntryMapping.byLocale).length} locales`,
+        "enrichConfigWithEntryMapping",
+        loggerPath,
+    );
+
+    try {
+        const config = JSON.parse(fs.readFileSync(configFilePath, "utf-8"));
+        config.__entryMapping__ = { old: oldEntryMapping, new: newEntryMapping };
+        fs.writeFileSync(configFilePath, JSON.stringify(config), "utf-8");
+    } catch (err) {
+        console.error("Failed to write entry mapping into update config:", err);
+        writeLogEntry(`Failed to write __entryMapping__ into ${configFilePath}: ${(err as Error)?.message}`, "enrichConfigWithEntryMapping", loggerPath);
+        return;
+    }
+
+    writeLogEntry(`Entry mapping enriched into config for iteration ${iteration}`, "enrichConfigWithEntryMapping", loggerPath);
+};
+
 /**
  * Ensures an update config file exists for this iteration and returns its path.
  * Used when there are asset updates but no entry updates produced a config, so

api/src/utils/locale-migration.utils.ts

@@ -64,6 +64,38 @@ export const isFullMigrationForLocale = (
   return !getMigratedLocales(project).includes(localeCode);
 };
 
+/**
+ * Extracts destination locale codes that were ACTUALLY targeted by a delta
+ * run, from an `updated-entries.json` config object.
+ *
+ * Per-entry keys in that config are `${csUid}::${localeCode}` (see
+ * `removeEntriesFromDatabase` in entry-update.utils.ts) — this reads the
+ * locale suffix back out. Bookkeeping keys added by the enrich* helpers
+ * (`__assetMapping__`, `__entryMapping__`, `__assetUpdates__`) are skipped.
+ *
+ * This exists to fix a bug where a locale got marked "migrated" as soon as
+ * ANY locale finished a delta run, instead of only the locale(s) that run
+ * actually processed — which permanently skipped locales configured ahead of
+ * when they were meant to be migrated (see `runCli.service.ts`).
+ */
+export const extractLocalesFromUpdateConfig = (
+  config: Record<string, any> | null | undefined,
+): string[] => {
+  if (!config || typeof config !== 'object') return [];
+  const locales = new Set<string>();
+  for (const [ctKey, entries] of Object.entries(config)) {
+    if (ctKey.startsWith('__')) continue;
+    if (!entries || typeof entries !== 'object') continue;
+    for (const entryKey of Object.keys(entries)) {
+      const sep = entryKey.lastIndexOf('::');
+      if (sep === -1) continue;
+      const locale = entryKey.slice(sep + 2);
+      if (locale) locales.add(locale);
+    }
+  }
+  return Array.from(locales);
+};
+
 /**
  * Set-union the given locales into project.migrated_locales and persist.
  * Idempotent.