module: synchronously load most ES modules

benchmark/esm/startup-esm-graph.js

@@ -0,0 +1,66 @@
+'use strict';
+
+const common = require('../common');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const tmpdir = require('../../test/common/tmpdir');
+
+const bench = common.createBenchmark(main, {
+  modules: ['0250', '0500', '1000', '2000'],
+  n: [30],
+});
+
+const BRANCHING_FACTOR = 10;
+
+function prepare(count) {
+  tmpdir.refresh();
+  const dir = tmpdir.resolve('esm-graph');
+  fs.mkdirSync(dir, { recursive: true });
+
+  // Create a tree-shaped ESM graph with a branching factor of 10.
+  // The root (mod0) plus `count` additional modules are created in BFS order.
+  // Module i imports modules BRANCHING_FACTOR*i+1 through
+  // BRANCHING_FACTOR*i+BRANCHING_FACTOR (capped at count), so the graph is a
+  // complete 10-ary tree rooted at mod0.
+  const total = count + 1;
+  for (let i = 0; i < total; i++) {
+    const children = [];
+    for (let c = 1; c <= BRANCHING_FACTOR; c++) {
+      const child = BRANCHING_FACTOR * i + c;
+      if (child < total) {
+        children.push(`import './mod${child}.mjs';`);
+      }
+    }
+    const content = children.join('\n') + (children.length ? '\n' : '') +
+      `export const value${i} = ${i};\n`;
+    fs.writeFileSync(path.join(dir, `mod${i}.mjs`), content);
+  }
+
+  return path.join(dir, 'mod0.mjs');
+}
+
+function main({ n, modules }) {
+  const entry = prepare(Number(modules));
+  const cmd = process.execPath || process.argv[0];
+  const warmup = 3;
+  const state = { finished: -warmup };
+
+  while (state.finished < n) {
+    const child = spawnSync(cmd, [entry]);
+    if (child.status !== 0) {
+      console.log('---- STDOUT ----');
+      console.log(child.stdout.toString());
+      console.log('---- STDERR ----');
+      console.log(child.stderr.toString());
+      throw new Error(`Child process stopped with exit code ${child.status}`);
+    }
+    state.finished++;
+    if (state.finished === 0) {
+      bench.start();
+    }
+    if (state.finished === n) {
+      bench.end(n);
+    }
+  }
+}

lib/internal/modules/esm/loader.js

@@ -369,6 +369,37 @@ class ModuleLoader {
     return { wrap: job.module, namespace: job.runSync(parent).namespace };
   }
 
+  /**
+   * Synchronously load and evaluate the entry point module.
+   * This avoids creating any promises when no TLA is present
+   * and no async customization hooks are registered.
+   * @param {string} url The URL of the entry point module.
+   * @returns {{ module: ModuleWrap, completed: boolean }} The entry module and whether
+   *   evaluation completed synchronously. When false, the caller should fall back to
+   *   async evaluation (TLA detected).
+   */
+  importSyncForEntryPoint(url) {
+    return onImport.traceSync(() => {
+      const request = { specifier: url, phase: kEvaluationPhase, attributes: kEmptyObject, __proto__: null };
+      const job = this.getOrCreateModuleJob(undefined, request, kImportInImportedESM);
+      if (getOptionValue('--inspect-brk')) {
+        const { callAndPauseOnStart } = internalBinding('inspector');
+        callAndPauseOnStart(job.module.instantiate, job.module);
+      } else {
+        job.module.instantiate();
+      }
+      if (job.module.hasAsyncGraph) {
+        return { __proto__: null, module: job.module, completed: false };
+      }
+      job.runSync();
+      return { __proto__: null, module: job.module, completed: true };
+    }, {
+      __proto__: null,
+      parentURL: undefined,
+      url,
+    });
+  }
+
   /**
    * Check invariants on a cached module job when require()'d from ESM.
    * @param {string} specifier The first parameter of require().
@@ -564,10 +595,15 @@ class ModuleLoader {
     }
 
     const { ModuleJob, ModuleJobSync } = require('internal/modules/esm/module_job');
-    // TODO(joyeecheung): use ModuleJobSync for kRequireInImportedCJS too.
-    const ModuleJobCtor = (requestType === kImportInRequiredESM ? ModuleJobSync : ModuleJob);
     const isMain = (parentURL === undefined);
     const inspectBrk = (isMain && getOptionValue('--inspect-brk'));
+    // Use ModuleJobSync whenever we're on the main thread (not the async loader hook worker),
+    // except for kRequireInImportedCJS (TODO: consolidate that case too) and --inspect-brk
+    // (which needs the async ModuleJob to pause on the first line).
+    // TODO(joyeecheung): use ModuleJobSync for kRequireInImportedCJS too.
+    const ModuleJobCtor = (!this.isForAsyncLoaderHookWorker && !inspectBrk &&
+                           requestType !== kRequireInImportedCJS) ?
+      ModuleJobSync : ModuleJob;
     job = new ModuleJobCtor(
       this,
       url,
@@ -594,8 +630,9 @@ class ModuleLoader {
    */
   getOrCreateModuleJob(parentURL, request, requestType) {
     let maybePromise;
-    if (requestType === kRequireInImportedCJS || requestType === kImportInRequiredESM) {
-      // In these two cases, resolution must be synchronous.
+    if (!this.isForAsyncLoaderHookWorker) {
+      // On the main thread, always resolve synchronously;
+      // `resolveSync` coordinates with the async loader hook worker if needed.
       maybePromise = this.resolveSync(parentURL, request);
       assert(!isPromise(maybePromise));
     } else {

lib/internal/modules/esm/module_job.js

@@ -46,7 +46,7 @@ const {
   getSourceMapsSupport,
 } = require('internal/source_map/source_map_cache');
 const assert = require('internal/assert');
-const resolvedPromise = PromiseResolve();
+let resolvedPromise;
 const {
   setHasStartedUserESMExecution,
   urlToFilename,
@@ -126,6 +126,58 @@ const explainCommonJSGlobalLikeNotDefinedError = (e, url, hasTopLevelAwait) => {
   }
 };
 
+/**
+ * If `error` is a SyntaxError from V8 for a missing named export on a CJS module, rewrite its message to the friendlier
+ * "Named export '...' not found..." form. Must be called after `decorateErrorStack(error)` so that the arrow (source
+ * context with the import statement text) has been prepended to `error.stack`.
+ * @param {Error} error
+ * @param {ModuleWrap} module The parent module that triggered the instantiation.
+ * @param {boolean[]} commonJsDeps Per-request array indicating whether each dependency is a CJS module, aligned with
+ *   `module.getModuleRequests()`.
+ */
+const handleCJSNamedExportError = (error, module, commonJsDeps) => {
+  // TODO(@bcoe): Add source map support to exception that occurs as result
+  // of missing named export. This is currently not possible because
+  // stack trace originates in module_job, not the file itself. A hidden
+  // symbol with filename could be set in node_errors.cc to facilitate this.
+  if (!getSourceMapsSupport().enabled &&
+      StringPrototypeIncludes(error.message,
+                              ' does not provide an export named')) {
+    const splitStack = StringPrototypeSplit(error.stack, '\n', 2);
+    const { 1: childSpecifier, 2: name } = RegExpPrototypeExec(
+      /module '(.*)' does not provide an export named '(.+)'/,
+      error.message);
+    const moduleRequests = module.getModuleRequests();
+    let isCommonJS = false;
+    for (let i = 0; i < moduleRequests.length; ++i) {
+      if (moduleRequests[i].specifier === childSpecifier) {
+        isCommonJS = commonJsDeps[i];
+        break;
+      }
+    }
+    if (isCommonJS) {
+      const importStatement = splitStack[1];
+      // TODO(@ctavan): The original error stack only provides the single
+      // line which causes the error. For multi-line import statements we
+      // cannot generate an equivalent object destructuring assignment by
+      // just parsing the error stack.
+      const oneLineNamedImports = RegExpPrototypeExec(/{.*}/, importStatement);
+      const destructuringAssignment = oneLineNamedImports &&
+        RegExpPrototypeSymbolReplace(/\s+as\s+/g, oneLineNamedImports, ': ');
+      error.message = `Named export '${name}' not found. The requested module` +
+        ` '${childSpecifier}' is a CommonJS module, which may not support` +
+        ' all module.exports as named exports.\nCommonJS modules can ' +
+        'always be imported via the default export, for example using:' +
+        `\n\nimport pkg from '${childSpecifier}';\n${
+          destructuringAssignment ?
+            `const ${destructuringAssignment} = pkg;\n` : ''}`;
+      const newStack = StringPrototypeSplit(error.stack, '\n');
+      newStack[3] = `SyntaxError: ${error.message}`;
+      error.stack = ArrayPrototypeJoin(newStack, '\n');
+    }
+  }
+};
+
 class ModuleJobBase {
   constructor(loader, url, importAttributes, phase, isMain, inspectBrk) {
     assert(typeof phase === 'number');
@@ -325,56 +377,16 @@ class ModuleJob extends ModuleJobBase {
       } else {
         this.module.instantiate();
       }
-    } catch (e) {
-      decorateErrorStack(e);
-      // TODO(@bcoe): Add source map support to exception that occurs as result
-      // of missing named export. This is currently not possible because
-      // stack trace originates in module_job, not the file itself. A hidden
-      // symbol with filename could be set in node_errors.cc to facilitate this.
-      if (!getSourceMapsSupport().enabled &&
-          StringPrototypeIncludes(e.message,
-                                  ' does not provide an export named')) {
-        const splitStack = StringPrototypeSplit(e.stack, '\n', 2);
-        const { 1: childSpecifier, 2: name } = RegExpPrototypeExec(
-          /module '(.*)' does not provide an export named '(.+)'/,
-          e.message);
-        const moduleRequests = this.module.getModuleRequests();
-        let isCommonJS = false;
-        for (let i = 0; i < moduleRequests.length; ++i) {
-          if (moduleRequests[i].specifier === childSpecifier) {
-            isCommonJS = this.commonJsDeps[i];
-            break;
-          }
-        }
-
-        if (isCommonJS) {
-          const importStatement = splitStack[1];
-          // TODO(@ctavan): The original error stack only provides the single
-          // line which causes the error. For multi-line import statements we
-          // cannot generate an equivalent object destructuring assignment by
-          // just parsing the error stack.
-          const oneLineNamedImports = RegExpPrototypeExec(/{.*}/, importStatement);
-          const destructuringAssignment = oneLineNamedImports &&
-            RegExpPrototypeSymbolReplace(/\s+as\s+/g, oneLineNamedImports, ': ');
-          e.message = `Named export '${name}' not found. The requested module` +
-            ` '${childSpecifier}' is a CommonJS module, which may not support` +
-            ' all module.exports as named exports.\nCommonJS modules can ' +
-            'always be imported via the default export, for example using:' +
-            `\n\nimport pkg from '${childSpecifier}';\n${
-              destructuringAssignment ?
-                `const ${destructuringAssignment} = pkg;\n` : ''}`;
-          const newStack = StringPrototypeSplit(e.stack, '\n');
-          newStack[3] = `SyntaxError: ${e.message}`;
-          e.stack = ArrayPrototypeJoin(newStack, '\n');
-        }
-      }
-      throw e;
+    } catch (error) {
+      decorateErrorStack(error);
+      handleCJSNamedExportError(error, this.module, this.commonJsDeps);
+      throw error;
     }
 
     for (const dependencyJob of jobsInGraph) {
       // Calling `this.module.instantiate()` instantiates not only the
       // ModuleWrap in this module, but all modules in the graph.
-      dependencyJob.instantiated = resolvedPromise;
+      dependencyJob.instantiated = resolvedPromise ??= PromiseResolve();
     }
   }
 
@@ -445,12 +457,14 @@ class ModuleJob extends ModuleJobBase {
 
 /**
  * This is a fully synchronous job and does not spawn additional threads in any way.
- * All the steps are ensured to be synchronous and it throws on instantiating
- * an asynchronous graph. It also disallows CJS <-> ESM cycles.
+ * Loading and linking are always synchronous. Evaluation via runSync() throws on an
+ * asynchronous graph; evaluation via run() falls back to async for top-level await.
+ * It also disallows CJS <-> ESM cycles.
  *
- * This is used for ES modules loaded via require(esm). Modules loaded by require() in
- * imported CJS are handled by ModuleJob with the isForRequireInImportedCJS set to true instead.
- * The two currently have different caching behaviors.
+ * Used for all ES module imports on the main thread, regardless of how the import was
+ * triggered (entry point, import(), require(esm), --import, etc.).
+ * Modules loaded by require() in imported CJS are handled by ModuleJob with the
+ * isForRequireInImportedCJS set to true instead. The two currently have different caching behaviors.
  * TODO(joyeecheung): consolidate this with the isForRequireInImportedCJS variant of ModuleJob.
  */
 class ModuleJobSync extends ModuleJobBase {
@@ -491,32 +505,55 @@ class ModuleJobSync extends ModuleJobBase {
     return PromiseResolve(this.module);
   }
 
-  async run() {
+  async run(isEntryPoint = false) {
     assert(this.phase === kEvaluationPhase);
-    // This path is hit by a require'd module that is imported again.
     const status = this.module.getStatus();
     debug('ModuleJobSync.run()', status, this.module);
     // If the module was previously required and errored, reject from import() again.
     if (status === kErrored) {
       throw this.module.getError();
-    } else if (status > kInstantiated) {
+    }
+    if (status > kInstantiated) {
+      // Already evaluated (e.g. previously require()'d and now import()'d again).
       if (this.evaluationPromise) {
         await this.evaluationPromise;
       }
       return { __proto__: null, module: this.module };
-    } else if (status === kInstantiated) {
-      // The evaluation may have been canceled because instantiate() detected TLA first.
-      // But when it is imported again, it's fine to re-evaluate it asynchronously.
+    }
+    if (status < kInstantiated) {
+      // Fresh module: instantiate it now (links were already resolved synchronously in constructor)
+      try {
+        this.module.instantiate();
+      } catch (error) {
+        decorateErrorStack(error);
+        handleCJSNamedExportError(error, this.module, this.commonJsDeps);
+        throw error;
+      }
+    }
+    // `status === kInstantiated`: either just instantiated above, or previously instantiated
+    // but evaluation was deferred (e.g. TLA detected by a prior `runSync()` call)
+    if (isEntryPoint) {
+      globalThis[entry_point_module_private_symbol] = this.module;
+    }
+    setHasStartedUserESMExecution();
+    if (this.module.hasAsyncGraph) {
+      // Has top-level `await`: fall back to async evaluation
       const timeout = -1;
       const breakOnSigint = false;
       this.evaluationPromise = this.module.evaluate(timeout, breakOnSigint);
       await this.evaluationPromise;
       this.evaluationPromise = undefined;
       return { __proto__: null, module: this.module };
     }
-
-    assert.fail('Unexpected status of a module that is imported again after being required. ' +
-                `Status = ${status}`);
+    // No top-level `await`: evaluate synchronously
+    const filename = urlToFilename(this.url);
+    try {
+      this.module.evaluateSync(filename, undefined);
+    } catch (evaluateError) {
+      explainCommonJSGlobalLikeNotDefinedError(evaluateError, this.module.url, this.module.hasTopLevelAwait);
+      throw evaluateError;
+    }
+    return { __proto__: null, module: this.module };
   }
 
   runSync(parent) {

lib/internal/modules/run_main.js

@@ -19,6 +19,7 @@ const {
 const {
   privateSymbols: {
     entry_point_promise_private_symbol,
+    entry_point_module_private_symbol,
   },
 } = internalBinding('util');
 /**
@@ -156,6 +157,19 @@ function executeUserEntryPoint(main = process.argv[1]) {
     const mainPath = resolvedMain || main;
     const mainURL = getOptionValue('--entry-url') ? new URL(mainPath, getCWDURL()) : pathToFileURL(mainPath);
 
+    // When no async hooks are needed, try the fully synchronous path first.
+    // This avoids creating any promises during startup.
+    if (getOptionValue('--experimental-loader').length === 0 &&
+        getOptionValue('--import').length === 0) {
+      const cascadedLoader = require('internal/modules/esm/loader').getOrInitializeCascadedLoader();
+      const { module: entryModule, completed } = cascadedLoader.importSyncForEntryPoint(mainURL.href);
+      globalThis[entry_point_module_private_symbol] = entryModule;
+      if (completed) {
+        return;
+      }
+      // TLA detected: fall through to async path.
+    }
+
     runEntryPointWithESMLoader((cascadedLoader) => {
       // Note that if the graph contains unsettled TLA, this may never resolve
       // even after the event loop stops running.