feat(lexer): prototype versioned detection baselines

Makefile

@@ -6,7 +6,7 @@ WASM_OPT ?= ../binaryen/bin/wasm-opt
 lib/lexer.wat: lib/lexer.wasm
 	$(WASM2WAT) lib/lexer.wasm -o lib/lexer.wat
 
-lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c | lib/
+lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c src/reexport-analysis.h | lib/
 	node build/wasm.js --docker
 
 lib/:

README.md

@@ -16,7 +16,7 @@ This project is used in Node.js core for detecting the named exports available w
 
 PRs will be accepted and upstreamed for parser bugs, performance improvements or new syntax support only.
 
-_Default detection patterns for this project are **frozen**_. This is because adding any new export detection patterns would result in fragmented backwards-compatibility. Specifically, it would be very difficult to figure out why an ES module named export for CommonJS might work in newer Node.js versions but not older versions. This problem would only be discovered downstream of module authors, with the fix for module authors being to then have to understand which patterns in this project provide full backwards-compatibily. Rather, by fully freezing the detected patterns, if it works in any Node.js version it will work in any other. Build tools can also reliably treat the supported syntax for this project as a part of their output target for ensuring syntax support.
+_Detection patterns for the `legacy` baseline are **frozen**_. This is because adding any new export detection patterns would result in fragmented backwards-compatibility. Specifically, it would be very difficult to figure out why an ES module named export for CommonJS might work in newer Node.js versions but not older versions. This problem would only be discovered downstream of module authors, with the fix for module authors being to then have to understand which patterns in this project provide full backwards-compatibily. Rather, by fully freezing the detected patterns, if it works in any Node.js version it will work in any other. Build tools can also reliably treat the supported syntax for this project as a part of their output target for ensuring syntax support.
 
 ### Usage
 
@@ -29,30 +29,16 @@ For use in CommonJS:
 ```js
 const { parse } = require('cjs-module-lexer');
 
-// `init` return a promise for parity with the ESM API, but you do not have to call it
+// Parsing initializes Wasm on demand. Await init() to initialize it in advance.
 
-const { exports, reexports } = parse(`
-  // named exports detection
-  module.exports.a = 'a';
-  (function () {
-    exports.b = 'b';
-  })();
-  Object.defineProperty(exports, 'c', { value: 'c' });
-  /* exports.d = 'not detected'; */
-
-  // reexports detection
-  if (maybe) module.exports = require('./dep1.js');
-  if (another) module.exports = require('./dep2.js');
-
-  // literal exports assignments
-  module.exports = { a, b: c, d, 'e': f }
-
-  // __esModule detection
-  Object.defineProperty(module.exports, '__esModule', { value: true })
+const { exports, reexports, complete } = parse(`
+  if (mode) module.exports = { own: 1, ...require('./dep1.js') };
+  else module.exports = { own: 2, ...require('./dep2.js') };
 `);
 
-// exports === ['a', 'b', 'c', '__esModule']
-// reexports === ['./dep1.js', './dep2.js']
+// exports: ['own']
+// reexports: [['./dep1.js'], ['./dep2.js']]
+// complete: true
 ```
 
 When using the ESM version, Wasm is supported instead:
@@ -85,6 +71,51 @@ const { exports } = parse(
 // exports === ['b']
 ```
 
+### Detection baselines (proof of concept)
+
+The default `reexports` value stays flat for one distinct final dependency list.
+Different conditional lists produce an array of arrays.
+Each inner array contains dependencies combined in one final export value.
+The outer array contains alternative final values.
+Identical lists are deduplicated before choosing the return shape.
+Conditional nesting changes the return type and requires a major release.
+
+```js
+const { reexports, complete } = parse(`
+  if (production) module.exports = {...require('./a.cjs'), ...require('./b.cjs')};
+  else module.exports = {...require('./c.cjs'), ...require('./d.cjs')};
+`);
+// reexports: [['./a.cjs', './b.cjs'], ['./c.cjs', './d.cjs']]
+// complete: true
+```
+
+Sequential replacements retain only the final value.
+Pure guards remove unreachable branches. Unknown guards retain their possible alternatives.
+Object spreads of direct `require` calls combine dependencies in one group.
+The `exports` array contains detected literal names from surviving values.
+
+Flow analysis runs only after the lexer detects more than one `module.exports` replacement.
+The counter ignores comments, strings, property writes, and comparisons.
+With fewer replacements, existing detection patterns run and their reexports stay flat.
+The optional third argument selects a versioned detection baseline.
+Select `baseline: 'legacy'` to disable flow analysis, or `baseline: 'flow-v1'` to enable it.
+These baselines preserve the selected detection extensions when flow analysis does not run.
+Extended mode disables flow analysis unless `baseline: 'flow-v1'` is explicit.
+A baseline can combine detection rules without an option for each rule.
+
+The C lexer compiled to Wasm owns flow analysis.
+The JavaScript entry point invokes it only for competing replacements.
+Use valid CommonJS source with standard CommonJS bindings and an ordinary writable `module.exports` property.
+Analysis describes successful synchronous completion and does not execute source or resolve dependencies.
+Opaque reads and dependency execution invalidate earlier exposed values. A fresh replacement can establish a known value again.
+
+Results include `complete` only when flow analysis runs.
+Unsupported syntax, effects, bindings, filtered string literals, and analysis limits make it false.
+An incomplete result contains confirmed alternatives and omits unknown values.
+Token, nesting, outcome, and work limits bound the analyzer.
+Existing lexer errors still apply.
+
+
 ### Extended detection
 
 The third argument selects an opt-in detection mode:

build/Makefile

@@ -14,7 +14,7 @@ WASM_EXTRA_LDFLAGS += -Wl,--export=parseCJSWithOptions
 WASM_EXTRA_LDFLAGS += -Wl,--export=e,--export=re,--export=es,--export=ee
 WASM_EXTRA_LDFLAGS += -Wl,--export=rre,--export=ree,--export=res,--export=ru,--export=us,--export=ue
 
-lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c
+lib/lexer.wasm: include-wasm/cjs-module-lexer.h src/lexer.c src/reexport-analysis.h
 	@mkdir -p lib
 	$(WASM_CC) $(WASM_CFLAGS) $(WASM_TARGET) $(WASM_EXTRA_CFLAGS) \
         src/lexer.c -o lib/lexer.wasm \

include-wasm/cjs-module-lexer.h

@@ -77,6 +77,8 @@ uint32_t ee () {
 }
 // getReexportStart
 uint32_t res () {
+  // A null slice starts a group; Wasm exposes this sentinel as signed -1.
+  if (reexport_read_head->start == NULL) return UINT32_MAX;
   return reexport_read_head->start - source;
 }
 // getReexportEnd

lexer.d.ts

@@ -1,13 +1,24 @@
 export interface Exports {
+  exports: string[];
+  /** Flat dependencies, or alternative lists when conditional final dependencies differ. */
+  reexports: string[] | string[][];
+  /** Present when competing replacements are analyzed. False for unsupported effects, filtered names, or limits. */
+  complete?: boolean;
+}
+
+export interface LegacyExports {
   exports: string[];
   reexports: string[];
 }
 
 export interface ParseOptions {
   allowMinifiedEnumerable?: boolean;
   mode?: 'default' | 'extended';
+  baseline?: 'legacy' | 'flow-v1';
 }
 
+export declare function parse(source: string, name: string | undefined, options: ParseOptions & { baseline: 'legacy' }): LegacyExports;
+export declare function parse(source: string, name: string | undefined, options: ParseOptions & { mode: 'extended'; baseline?: 'legacy' }): LegacyExports;
 export declare function parse(source: string, name?: string, options?: ParseOptions): Exports;
 export declare function init(): Promise<void>;
 export declare function initSync(): void;

lexer.js

@@ -17,7 +17,8 @@ let openTokenDepth,
   lastStarExportSpecifier,
   _exports,
   unsafeGetters,
-  reexports;
+  reexports,
+  moduleExportCount;
 
 function resetState () {
   openTokenDepth = 0;
@@ -33,6 +34,7 @@ function resetState () {
   _exports = new Set();
   unsafeGetters = new Set();
   reexports = new Set();
+  moduleExportCount = 0;
 }
 
 // RequireType
@@ -66,6 +68,11 @@ function parseCJS (source, name = '@', options) {
   if (mode !== undefined && mode !== 'default' && mode !== 'extended')
     throw new TypeError('Expected options.mode to be "default" or "extended".');
   const minifiedEnumerable = options ? options.allowMinifiedEnumerable === true : false;
+  const baseline = options?.baseline;
+  if (baseline !== undefined && typeof baseline !== 'string')
+    throw new TypeError('Detection baseline must be a string');
+  if (baseline !== undefined && baseline !== 'legacy' && baseline !== 'flow-v1')
+    throw new RangeError('Unknown detection baseline: ' + baseline);
   resetState();
   allowMinifiedEnumerable = minifiedEnumerable;
   extended = mode === 'extended';
@@ -78,6 +85,11 @@ function parseCJS (source, name = '@', options) {
     error.loc = pos;
     throw error;
   }
+  if (moduleExportCount > 1 && baseline !== 'legacy' && (mode !== 'extended' || baseline === 'flow-v1')) {
+    resetState();
+    return require('./dist/lexer.js').parse(source, name,
+      { baseline, mode, allowMinifiedEnumerable: minifiedEnumerable });
+  }
   const result = { exports: [..._exports].filter(expt => expt !== undefined && !unsafeGetters.has(expt)), reexports: [...reexports].filter(reexpt => reexpt !== undefined) };
   resetState();
   return result;
@@ -999,6 +1011,8 @@ function tryParseExportsDotAssign (assign) {
     // module.exports =
     case 61/*=*/: {
       if (assign) {
+        if (moduleExportCount < 2 && source.charCodeAt(pos + 1) !== 61)
+          moduleExportCount++;
         if (!extended && reexports.size)
           reexports = new Set();
         pos++;
@@ -1020,6 +1034,13 @@ function tryParseExportsDotAssign (assign) {
         if (ch === 114/*r*/)
           tryParseRequire(ExportAssign);
       }
+      break;
+    }
+    case 38/*&*/:
+    case 124/*|*/:
+    case 63/*?*/: {
+      if (assign && moduleExportCount < 2 && source.charCodeAt(pos + 1) === ch &&
+          source.charCodeAt(pos + 2) === 61) moduleExportCount++;
     }
   }
   pos = revertPos;
@@ -1890,8 +1911,6 @@ function isExpressionTerminator (curPos) {
   return false;
 }
 
-const initPromise = Promise.resolve();
-
-module.exports.init = () => initPromise;
-module.exports.initSync = () => {};
+module.exports.init = () => require('./dist/lexer.js').init();
+module.exports.initSync = () => require('./dist/lexer.js').initSync();
 module.exports.parse = parseCJS;

src/lexer.c

@@ -9,6 +9,7 @@ const uint16_t* EMPTY_CHAR = &__empty_char;
 enum TokenType { ClassToken = 1, ExportObjectToken = 2 };
 const uint16_t* source;
 
+
 bool lastSlashWasDivision;
 uint16_t templateStackDepth;
 uint16_t openTokenDepth;
@@ -33,6 +34,10 @@ uint16_t* lastReexportEnd;
 uint32_t parse_error;
 uint32_t error = 0;
 uint32_t sourceLen;
+#ifdef __wasm__
+uint32_t moduleExportCount;
+#include "reexport-analysis.h"
+#endif
 
 uint16_t templateStack_[STACK_DEPTH];
 uint16_t* openTokenPosStack_[STACK_DEPTH];
@@ -85,6 +90,9 @@ uint32_t parseCJSWithOptions (uint16_t* _source, uint32_t _sourceLen, void (*_ad
   sourceLen = _sourceLen;
   allowMinifiedEnumerable = (options & 1) != 0;
   extended = (options & 2) != 0;
+#ifdef __wasm__
+  moduleExportCount = 0;
+#endif
   if (_addExport)
     addExport = _addExport;
   if (_addReexport)
@@ -303,6 +311,10 @@ uint32_t parseCJSWithOptions (uint16_t* _source, uint32_t _sourceLen, void (*_ad
   }
 
   // success
+#ifdef __wasm__
+  if ((options & 4) && moduleExportCount > 1)
+    return analyzeReexports();
+#endif
   return 0;
 }
 
@@ -1004,6 +1016,10 @@ void tryParseExportsDotAssign (bool assign) {
     // module.exports =
     case '=': {
       if (assign) {
+#ifdef __wasm__
+        if (moduleExportCount < 2 && pos[1] != '=')
+          moduleExportCount++;
+#endif
         if (!extended) clearReexports();
         pos++;
         ch = commentWhitespace();
@@ -1024,6 +1040,15 @@ void tryParseExportsDotAssign (bool assign) {
         if (ch == 'r')
           tryParseRequire(ExportAssign);
       }
+      break;
+    }
+    case '&':
+    case '|':
+    case '?': {
+#ifdef __wasm__
+      if (assign && moduleExportCount < 2 && end - pos >= 2 && pos[1] == ch && pos[2] == '=')
+        moduleExportCount++;
+#endif
     }
   }
   pos = revertPos;

src/lexer.js

@@ -12,8 +12,15 @@ export function parse (source, name = '@', options) {
   const mode = options ? options.mode : undefined;
   if (mode !== undefined && mode !== 'default' && mode !== 'extended')
     throw new TypeError('Expected options.mode to be "default" or "extended".');
-  const flags = (options && options.allowMinifiedEnumerable === true ? 1 : 0) |
-    (mode === 'extended' ? 2 : 0);
+  const minifiedEnumerable = options ? options.allowMinifiedEnumerable === true : false;
+  const baseline = options?.baseline;
+  if (baseline !== undefined && typeof baseline !== 'string')
+    throw new TypeError('Detection baseline must be a string');
+  if (baseline !== undefined && baseline !== 'legacy' && baseline !== 'flow-v1')
+    throw new RangeError('Unknown detection baseline: ' + baseline);
+  const flags = (minifiedEnumerable ? 1 : 0) |
+    (mode === 'extended' ? 2 : 0) |
+    (baseline !== 'legacy' && (mode !== 'extended' || baseline === 'flow-v1') ? 4 : 0);
   if (!wasm)
     initSync();
 
@@ -34,7 +41,8 @@ export function parse (source, name = '@', options) {
   else
     (isLE ? copyLE : copyBE)(source, new Uint16Array(memory, addr, len));
 
-  const err_code = wasm.parseCJSWithOptions(addr, len, 0, 0, 0, 0, flags);
+  const status = wasm.parseCJSWithOptions(addr, len, 0, 0, 0, 0, flags);
+  const err_code = status & 255;
 
   if (err_code) {
     const err = new Error(`Parse error ${name}${wasm.e()}:${source.slice(0, wasm.e()).split('\n').length}:${wasm.e() - source.lastIndexOf('\n', wasm.e() - 1)}`);
@@ -45,23 +53,51 @@ export function parse (source, name = '@', options) {
   }
 
   let exports = new Set(), reexports = new Set(), unsafeGetters;
+  const groups = status & 1024 ? [] : undefined;
+  let complete = !(status & 2048);
 
   while (wasm.rre()) {
-    const reexptStr = decode(source.slice(wasm.res(), wasm.ree()));
+    const start = wasm.res();
+    if (start === -1) {
+      reexports = new Set();
+      groups.push(reexports);
+      continue;
+    }
+    const reexptStr = decode(source.slice(start, wasm.ree()));
     if (reexptStr !== undefined)
       reexports.add(reexptStr);
+    else
+      complete = false;
   }
   while (wasm.ru()) {
     if (!unsafeGetters) unsafeGetters = new Set();
     unsafeGetters.add(decode(source.slice(wasm.us(), wasm.ue())));
   }
   while (wasm.re()) {
-    let exptStr = decode(source.slice(wasm.es(), wasm.ee()));
+    const text = source.slice(wasm.es(), wasm.ee());
+    let exptStr = status & 1024 && text.charCodeAt(0) >= 48 && text.charCodeAt(0) <= 57 ?
+      String(Number(text)) : decode(text);
     if (exptStr !== undefined && (!unsafeGetters || !unsafeGetters.has(exptStr)))
       exports.add(exptStr);
+    else if (exptStr === undefined)
+      complete = false;
   }
 
-  return { exports: [...exports], reexports: [...reexports] };
+  if (!(status & 1024)) return { exports: [...exports], reexports: [...reexports] };
+  if (groups.length === 1)
+    return { exports: [...exports], reexports: [...reexports], complete };
+  const alternatives = [];
+  const seen = new Set();
+  let hasDependencies = false;
+  for (const group of groups) {
+    const values = [...group];
+    const key = JSON.stringify(values);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    alternatives.push(values);
+    if (values.length) hasDependencies = true;
+  }
+  return { exports: [...exports], reexports: hasDependencies ? alternatives.length === 1 ? alternatives[0] : alternatives : [], complete };
 }
 
 function decode (str) {