test_runner: support test order randomization

PR-URL: #61747
Reviewed-By: Jacob Smith <jacob@frende.me>
Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
Reviewed-By: Ethan Arrowood <ethan@arrowood.dev>
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>

Author: pmarchini

doc/api/cli.md

@@ -2784,6 +2784,38 @@ changes:
 Configures the test runner to only execute top level tests that have the `only`
 option set. This flag is not necessary when test isolation is disabled.
 
+### `--test-random-seed`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Set the seed used to randomize test execution order. This applies to both test
+file execution order and queued tests within each file. Providing this flag
+enables randomization implicitly, even without `--test-randomize`.
+
+The value must be an integer between `0` and `4294967295`.
+
+This flag cannot be used with `--watch` or `--test-rerun-failures`.
+
+### `--test-randomize`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Randomize test execution order. This applies to both test file execution order
+and queued tests within each file. This can help detect tests that rely on
+shared state or execution order.
+
+The seed used for randomization is printed in the test summary and can be
+reused with `--test-random-seed`.
+
+For detailed behavior and examples, see
+[randomizing tests execution order][].
+
+This flag cannot be used with `--watch` or `--test-rerun-failures`.
+
 ### `--test-reporter`
 
 <!-- YAML
@@ -3702,6 +3734,8 @@ one is included in the list below.
 * `--test-isolation`
 * `--test-name-pattern`
 * `--test-only`
+* `--test-random-seed`
+* `--test-randomize`
 * `--test-reporter-destination`
 * `--test-reporter`
 * `--test-rerun-failures`
@@ -4286,6 +4320,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
 [module compile cache]: module.md#module-compile-cache
 [preloading asynchronous module customization hooks]: module.md#registration-of-asynchronous-customization-hooks
+[randomizing tests execution order]: test.md#randomizing-tests-execution-order
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection
 [running tests from the command line]: test.md#running-tests-from-the-command-line
 [scavenge garbage collector]: https://v8.dev/blog/orinoco-parallel-scavenger

doc/api/test.md

@@ -627,6 +627,94 @@ prevent shell expansion, which can reduce portability across systems.
 node --test "**/*.test.js" "**/*.spec.js"
 ```
 
+### Randomizing tests execution order
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+The test runner can randomize execution order to help detect
+order-dependent tests. When enabled, the runner randomizes both discovered
+test files and queued tests within each file. Use `--test-randomize` to
+enable this mode.
+
+```bash
+node --test --test-randomize
+```
+
+When randomization is enabled, the test runner prints the seed used for the run
+as a diagnostic message:
+
+```text
+Randomized test order seed: 12345
+```
+
+Use `--test-random-seed=<number>` to replay the same randomized order
+deterministically. Supplying `--test-random-seed` also enables randomization,
+so `--test-randomize` is optional when a seed is provided:
+
+```bash
+node --test --test-random-seed=12345
+```
+
+In most test files, randomization works automatically. One important exception
+is when subtests are awaited one by one. In that pattern, each subtest starts
+only after the previous one finishes, so the runner keeps declaration order
+instead of randomizing it.
+
+Example: this runs sequentially and is **not** randomized.
+
+```mjs
+import test from 'node:test';
+
+test('math', async (t) => {
+  for (const name of ['adds', 'subtracts', 'multiplies']) {
+    // Sequentially awaiting each subtest preserves declaration order.
+    await t.test(name, async () => {});
+  }
+});
+```
+
+```cjs
+const test = require('node:test');
+
+test('math', async (t) => {
+  for (const name of ['adds', 'subtracts', 'multiplies']) {
+    // Sequentially awaiting each subtest preserves declaration order.
+    await t.test(name, async () => {});
+  }
+});
+```
+
+Using suite-style APIs such as `describe()`/`it()` or `suite()`/`test()`
+still allows randomization, because sibling tests are enqueued together.
+
+Example: this remains eligible for randomization.
+
+```mjs
+import { describe, it } from 'node:test';
+
+describe('math', () => {
+  it('adds', () => {});
+  it('subtracts', () => {});
+  it('multiplies', () => {});
+});
+```
+
+```cjs
+const { describe, it } = require('node:test');
+
+describe('math', () => {
+  it('adds', () => {});
+  it('subtracts', () => {});
+  it('multiplies', () => {});
+});
+```
+
+`--test-randomize` and `--test-random-seed` are not supported with `--watch` mode.
+
 Matching files are executed as test files.
 More information on the test file execution can be found
 in the [test runner execution model][] section.
@@ -667,6 +755,10 @@ test runner functionality:
 * `--test-reporter` - Reporting is managed by the parent process
 * `--test-reporter-destination` - Output destinations are controlled by the parent
 * `--experimental-config-file` - Config file paths are managed by the parent
+* `--test-randomize` - Randomization is managed by the parent process and
+  propagated to child processes
+* `--test-random-seed` - Randomization seed is managed by the parent process and
+  propagated to child processes
 
 All other Node.js options from command line arguments, environment variables,
 and configuration files are inherited by the child processes.
@@ -1575,6 +1667,14 @@ changes:
       that specifies the index of the shard to run. This option is _required_.
     * `total` {number} is a positive integer that specifies the total number
       of shards to split the test files to. This option is _required_.
+  * `randomize` {boolean} Randomize execution order for test files and queued tests.
+    This option is not supported with `watch: true`.
+    **Default:** `false`.
+  * `randomSeed` {number} Seed used when randomizing execution order. If this
+    option is set, runs can replay the same randomized order deterministically,
+    and setting this option also enables randomization. The value must be an
+    integer between `0` and `4294967295`.
+    **Default:** `undefined`.
   * `rerunFailuresFilePath` {string} A file path where the test runner will
     store the state of the tests to allow rerunning only the failed tests on a next run.
     see \[Rerunning failed tests]\[] for more information.

doc/node-config-schema.json

@@ -524,6 +524,14 @@
           "type": "boolean",
           "description": "run tests with 'only' option set"
         },
+        "test-random-seed": {
+          "type": "number",
+          "description": "seed used to randomize test execution order"
+        },
+        "test-randomize": {
+          "type": "boolean",
+          "description": "run tests in a random order"
+        },
         "test-reporter": {
           "oneOf": [
             {
@@ -906,6 +914,14 @@
           "type": "boolean",
           "description": "run tests with 'only' option set"
         },
+        "test-random-seed": {
+          "type": "number",
+          "description": "seed used to randomize test execution order"
+        },
+        "test-randomize": {
+          "type": "boolean",
+          "description": "run tests in a random order"
+        },
         "test-reporter": {
           "oneOf": [
             {

doc/node.1

@@ -1363,6 +1363,22 @@ tests must satisfy \fBboth\fR requirements in order to be executed.
 Configures the test runner to only execute top level tests that have the \fBonly\fR
 option set. This flag is not necessary when test isolation is disabled.
 .
+.It Fl -test-random-seed
+Set the seed used to randomize test execution order.
+This applies to both test file execution order and queued tests within each file.
+Providing this flag enables randomization implicitly, even without
+\fB--test-randomize\fR.
+The value must be an integer between 0 and 4294967295.
+This flag cannot be used with \fB--watch\fR or \fB--test-rerun-failures\fR.
+.
+.It Fl -test-randomize
+Randomize test execution order.
+This applies to both test file execution order and queued tests within each file.
+This can help detect tests that rely on shared state or execution order.
+The seed used for randomization is printed in the test summary and can be
+reused with \fB--test-random-seed\fR.
+This flag cannot be used with \fB--watch\fR or \fB--test-rerun-failures\fR.
+.
 .It Fl -test-reporter
 A test reporter to use when running tests. See the documentation on
 test reporters for more details.
@@ -2040,6 +2056,10 @@ one is included in the list below.
 .It
 \fB--test-reporter-destination\fR
 .It
+\fB--test-randomize\fR
+.It
+\fB--test-random-seed\fR
+.It
 \fB--test-reporter\fR
 .It
 \fB--test-rerun-failures\fR

lib/internal/test_runner/runner.js

@@ -59,6 +59,7 @@ const {
   validateObject,
   validateOneOf,
   validateInteger,
+  validateUint32,
   validateString,
   validateStringArray,
 } = require('internal/validators');
@@ -84,6 +85,7 @@ const {
 const { FastBuffer } = require('internal/buffer');
 
 const {
+  createRandomSeed,
   convertStringToRegExp,
   countCompletedTest,
   kDefaultPattern,
@@ -105,12 +107,14 @@ const kIsolatedProcessName = Symbol('kIsolatedProcessName');
 const kFilterArgs = [
   '--test',
   '--experimental-test-coverage',
+  '--test-randomize',
   '--watch',
   '--experimental-default-config-file',
 ];
 const kFilterArgValues = [
   '--test-reporter',
   '--test-reporter-destination',
+  '--test-random-seed',
   '--experimental-config-file',
 ];
 const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
@@ -168,6 +172,8 @@ function getRunArgs(path, { forceExit,
                             argv: suppliedArgs,
                             execArgv,
                             rerunFailuresFilePath,
+                            randomize,
+                            randomSeed,
                             root: { timeout },
                             cwd }) {
   const processNodeOptions = getOptionsAsFlagsFromBinding();
@@ -209,6 +215,12 @@ function getRunArgs(path, { forceExit,
   if (rerunFailuresFilePath) {
     ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`);
   }
+  if (randomize) {
+    ArrayPrototypePush(runArgs, '--test-randomize');
+  }
+  if (randomSeed != null) {
+    ArrayPrototypePush(runArgs, `--test-random-seed=${randomSeed}`);
+  }
 
   ArrayPrototypePushApply(runArgs, execArgv);
 
@@ -646,6 +658,8 @@ function run(options = kEmptyObject) {
     lineCoverage = 0,
     branchCoverage = 0,
     functionCoverage = 0,
+    randomize: suppliedRandomize,
+    randomSeed: suppliedRandomSeed,
     execArgv = [],
     argv = [],
     cwd = process.cwd(),
@@ -674,6 +688,56 @@ function run(options = kEmptyObject) {
   if (globPatterns != null) {
     validateArray(globPatterns, 'options.globPatterns');
   }
+  if (suppliedRandomize != null) {
+    validateBoolean(suppliedRandomize, 'options.randomize');
+  }
+  if (suppliedRandomSeed != null) {
+    validateUint32(suppliedRandomSeed, 'options.randomSeed');
+  }
+  let randomize = suppliedRandomize;
+  let randomSeed = suppliedRandomSeed;
+
+  if (randomSeed != null) {
+    randomize = true;
+  }
+  if (watch) {
+    if (randomSeed != null) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomSeed',
+        randomSeed,
+        'is not supported with watch mode',
+      );
+    }
+    if (randomize) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomize',
+        randomize,
+        'is not supported with watch mode',
+      );
+    }
+  }
+  if (rerunFailuresFilePath) {
+    validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
+    // TODO(pmarchini): Support rerun-failures with randomization by
+    // persisting the randomization seed in the rerun state file.
+    if (randomSeed != null) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomSeed',
+        randomSeed,
+        'is not supported with rerun failures mode',
+      );
+    }
+    if (randomize) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options.randomize',
+        randomize,
+        'is not supported with rerun failures mode',
+      );
+    }
+  }
+  if (randomize) {
+    randomSeed ??= createRandomSeed();
+  }
 
   validateString(cwd, 'options.cwd');
 
@@ -683,10 +747,6 @@ function run(options = kEmptyObject) {
     );
   }
 
-  if (rerunFailuresFilePath) {
-    validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
-  }
-
   if (shard != null) {
     validateObject(shard, 'options.shard');
     // Avoid re-evaluating the shard object in case it's a getter
@@ -783,11 +843,18 @@ function run(options = kEmptyObject) {
     functionCoverage: functionCoverage,
     cwd,
     globalSetupPath,
+    randomize,
+    randomSeed,
   };
+
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
   const { isTestRunner } = globalOptions;
 
+  if (randomize) {
+    root.diagnostic(`Randomized test order seed: ${randomSeed}`);
+  }
+
   if (shard) {
     testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
   }
@@ -833,6 +900,8 @@ function run(options = kEmptyObject) {
     rerunFailuresFilePath,
     env,
     workerIdPool: isolation === 'process' ? workerIdPool : null,
+    randomize,
+    randomSeed,
   };
 
   if (isolation === 'process') {