From f87b7255cfea3dc5e8881fdebf9af3ea0312888d Mon Sep 17 00:00:00 2001 From: Alejandro Espa Date: Sun, 8 Feb 2026 22:52:20 +0100 Subject: [PATCH 1/2] feat: first commit flaky --- lib/internal/test_runner/harness.js | 3 ++- lib/internal/test_runner/reporter/tap.js | 10 +++++++--- lib/internal/test_runner/reporter/utils.js | 5 ++++- lib/internal/test_runner/test.js | 4 +++- lib/internal/test_runner/utils.js | 4 ++++ package-lock.json | 6 ++++++ 6 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 package-lock.json diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 6b3b13b2c88d65..ecee5cdefaa897 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -61,6 +61,7 @@ function createTestTree(rootTestOptions, globalOptions) { failed: 0, passed: 0, cancelled: 0, + flaky: 0, skipped: 0, todo: 0, topLevel: 0, @@ -377,7 +378,7 @@ function runInParentContext(Factory) { return run(name, options, fn, overrides); }; - ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => { + ArrayPrototypeForEach(['expectFailure', 'flaky', 'skip', 'todo', 'only'], (keyword) => { test[keyword] = (name, options, fn) => { const overrides = { __proto__: null, diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 01c698871b9134..595c96b48c8970 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -33,12 +33,12 @@ async function * tapReporter(source) { for await (const { type, data } of source) { switch (type) { case 'test:fail': { - yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure); + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; yield reportDetails(data.nesting, data.details, location); break; } case 'test:pass': - yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure); + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); yield reportDetails(data.nesting, data.details, null); break; case 'test:plan': @@ -65,7 +65,7 @@ async function * tapReporter(source) { } } -function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) { +function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) { let line = `${indent(nesting)}${status} ${testNumber}`; if (name) { @@ -78,6 +78,10 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { line += ' # EXPECTED FAILURE'; + //should we use flaky >=0 here? for always printing 0 retries + } else if (flaky !== undefined && flaky > 0) { + const retryText = flaky === 1 ? 're-try' : 're-tries'; + line += ` # FLAKY ${flaky} ${retryText}`; } line += '\n'; diff --git a/lib/internal/test_runner/reporter/utils.js b/lib/internal/test_runner/reporter/utils.js index d90040b9727aa2..4f0f888572c2d1 100644 --- a/lib/internal/test_runner/reporter/utils.js +++ b/lib/internal/test_runner/reporter/utils.js @@ -71,7 +71,7 @@ function formatError(error, indent) { function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') { let color = reporterColorMap[type] ?? colors.white; let symbol = reporterUnicodeSymbolMap[type] ?? ' '; - const { skip, todo, expectFailure } = data; + const { skip, todo, expectFailure, flaky } = data; const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; let title = `${data.name}${duration_ms}`; @@ -87,6 +87,9 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde } } else if (expectFailure !== undefined) { title += ` # EXPECTED FAILURE`; + } else if (flaky !== undefined && flaky > 0) { + const retryText = flaky === 1 ? 're-try' : 're-tries'; + title += ` # FLAKY ${flaky} ${retryText}`; } const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : ''; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index beeb49c1763473..dffa7f0980d876 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -85,6 +85,7 @@ const kTestTimeoutFailure = 'testTimeoutFailure'; const kExpectedFailure = 'expectedFailure'; const kHookFailure = 'hookFailed'; const kDefaultTimeout = null; +const kDefaultFlakyRetries = 20; const noop = FunctionPrototype; const kShouldAbort = Symbol('kShouldAbort'); const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']); @@ -497,7 +498,8 @@ class Test extends AsyncResource { super('Test'); let { fn, name, parent } = options; - const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options; + + const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options; if (typeof fn !== 'function') { fn = noop; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 5b53342933cdcb..30ded02c25c3de 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -401,6 +401,10 @@ function countCompletedTest(test, harness = test.root.harness) { } else { harness.counters.passed++; } + + if (test.flakyRetries > 0) { + harness.counters.flaky++; + } harness.counters.tests++; } diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000000000..93bc0662b7b834 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "node", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} From 2879fc75d4fb6dbe088066ca510c79f95616c45f Mon Sep 17 00:00:00 2001 From: Alejandro Espa Date: Mon, 23 Mar 2026 23:11:40 +0100 Subject: [PATCH 2/2] feat: flaky tests implemented --- doc/api/test.md | 102 ++++ lib/internal/test_runner/reporter/dot.js | 11 +- lib/internal/test_runner/reporter/junit.js | 131 ++++- lib/internal/test_runner/reporter/tap.js | 80 ++- lib/internal/test_runner/reporter/utils.js | 39 +- lib/internal/test_runner/test.js | 507 ++++++++++++------ lib/internal/test_runner/tests_stream.js | 4 + package-lock.json | 6 - test/fixtures/test-runner/output/flaky.js | 68 +++ .../test-runner/output/flaky.snapshot | 69 +++ test/test-runner/test-output-flaky.mjs | 11 + 11 files changed, 805 insertions(+), 223 deletions(-) delete mode 100644 package-lock.json create mode 100644 test/fixtures/test-runner/output/flaky.js create mode 100644 test/fixtures/test-runner/output/flaky.snapshot create mode 100644 test/test-runner/test-output-flaky.mjs diff --git a/doc/api/test.md b/doc/api/test.md index 40fb08d0d5b181..966d70dd9e3875 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -275,6 +275,56 @@ it.todo('should do the thing', { expectFailure: true }, () => { }); ``` +## Flaky tests + + + +This flag causes a test or suite to be re-run a number of times until it +either passes or has not passed after the final re-try. + +When `flaky` is `true`, the test harness re-tries the test up to the default +number of times (20), inclusive. + +When `flaky` is a positive integer, the test harness re-tries the test up to +the specified number of times, inclusive. + +When `flaky` is falsy (the default), the test harness does not re-try the test. + +When both a suite and an included test specify the `flaky` flag, the +test's `flaky` value wins. + +```js +it.flaky('should do something', () => { + // This test will be retried up to 20 times if it fails +}); + +it('may take several times', { flaky: true }, () => { + // Also retries up to 20 times +}); + +it('may also take several times', { flaky: 5 }, () => { + // Retries up to 5 times +}); + +describe.flaky('flaky suite', () => { + it('inherits flaky from suite', () => { + // Retried up to 20 times (inherited from suite) + }); + + it('not flaky', { flaky: false }, () => { + // Not retried, overrides suite setting + }); +}); +``` + +When a test marked `flaky` passes after retries, the number of re-tries taken +is reported with that test. + +`skip` and `todo` take precedence over `flaky`. + ## `describe()` and `it()` aliases Suites and tests can also be written using the `describe()` and `it()` @@ -1649,6 +1699,16 @@ added: Shorthand for marking a suite as `only`. This is the same as [`suite([name], { only: true }[, fn])`][suite options]. +## `suite.flaky([name][, options][, fn])` + + + +Shorthand for marking a suite as flaky. This is the same as +[`suite([name], { flaky: true }[, fn])`][suite options]. + ## `test([name][, options][, fn])` + +Shorthand for marking a test as flaky, +same as [`test([name], { flaky: true }[, fn])`][it options]. + ## `describe([name][, options][, fn])` Alias for [`suite()`][]. @@ -1782,6 +1857,16 @@ added: Shorthand for marking a suite as `only`. This is the same as [`describe([name], { only: true }[, fn])`][describe options]. +## `describe.flaky([name][, options][, fn])` + + + +Shorthand for marking a suite as flaky. This is the same as +[`describe([name], { flaky: true }[, fn])`][describe options]. + ## `it([name][, options][, fn])` + +Shorthand for marking a test as flaky, +same as [`it([name], { flaky: true }[, fn])`][it options]. + ## `before([fn][, options])` \n`; @@ -44,21 +56,34 @@ function treeToXML(tree) { const attrsString = ArrayPrototypeJoin( ArrayPrototypeMap( ObjectEntries(attrs), - ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`), - ' '); + ({ 0: key, 1: value }) => `${key}="${escapeAttribute(String(value))}"`, + ), + ' ', + ); if (!children?.length) { return `${indent}<${tag} ${attrsString}/>\n`; } - const childrenString = ArrayPrototypeJoin(ArrayPrototypeMap(children ?? [], treeToXML), ''); + const childrenString = ArrayPrototypeJoin( + ArrayPrototypeMap(children ?? [], treeToXML), + '', + ); return `${indent}<${tag} ${attrsString}>\n${childrenString}${indent}\n`; } function isFailure(node) { - return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || node?.attrs?.failures; + return ( + (node?.children && + ArrayPrototypeSome(node.children, (c) => c.tag === 'failure')) || + node?.attrs?.failures + ); } function isSkipped(node) { - return (node?.children && ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || node?.attrs?.skipped; + return ( + (node?.children && + ArrayPrototypeSome(node.children, (c) => c.tag === 'skipped')) || + node?.attrs?.skipped + ); } module.exports = async function* junitReporter(source) { @@ -93,25 +118,42 @@ module.exports = async function* junitReporter(source) { case 'test:pass': case 'test:fail': { if (!currentSuite) { - startTest({ __proto__: null, data: { __proto__: null, name: 'root', nesting: 0 } }); + startTest({ + __proto__: null, + data: { __proto__: null, name: 'root', nesting: 0 }, + }); } - if (currentSuite.attrs.name !== event.data.name || - currentSuite.nesting !== event.data.nesting) { + if ( + currentSuite.attrs.name !== event.data.name || + currentSuite.nesting !== event.data.nesting + ) { startTest(event); } const currentTest = currentSuite; if (currentSuite?.nesting === event.data.nesting) { currentSuite = currentSuite.parent; } - currentTest.attrs.time = NumberPrototypeToFixed(event.data.details.duration_ms / 1000, 6); - const nonCommentChildren = ArrayPrototypeFilter(currentTest.children, (c) => c.comment == null); + currentTest.attrs.time = NumberPrototypeToFixed( + event.data.details.duration_ms / 1000, + 6, + ); + const nonCommentChildren = ArrayPrototypeFilter( + currentTest.children, + (c) => c.comment == null, + ); if (nonCommentChildren.length > 0) { currentTest.tag = 'testsuite'; currentTest.attrs.disabled = 0; currentTest.attrs.errors = 0; currentTest.attrs.tests = nonCommentChildren.length; - currentTest.attrs.failures = ArrayPrototypeFilter(currentTest.children, isFailure).length; - currentTest.attrs.skipped = ArrayPrototypeFilter(currentTest.children, isSkipped).length; + currentTest.attrs.failures = ArrayPrototypeFilter( + currentTest.children, + isFailure, + ).length; + currentTest.attrs.skipped = ArrayPrototypeFilter( + currentTest.children, + isSkipped, + ).length; currentTest.attrs.hostname = HOSTNAME; } else { currentTest.tag = 'testcase'; @@ -121,14 +163,46 @@ module.exports = async function* junitReporter(source) { } if (event.data.skip) { ArrayPrototypePush(currentTest.children, { - __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', - attrs: { __proto__: null, type: 'skipped', message: event.data.skip }, + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'skipped', + attrs: { + __proto__: null, + type: 'skipped', + message: event.data.skip, + }, }); } if (event.data.todo) { ArrayPrototypePush(currentTest.children, { - __proto__: null, nesting: event.data.nesting + 1, tag: 'skipped', - attrs: { __proto__: null, type: 'todo', message: event.data.todo }, + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'skipped', + attrs: { + __proto__: null, + type: 'todo', + message: event.data.todo, + }, + }); + } + if (event.data.flakyRetriedCount > 0) { + ArrayPrototypePush(currentTest.children, { + __proto__: null, + nesting: event.data.nesting + 1, + tag: 'properties', + attrs: { __proto__: null }, + children: [ + { + __proto__: null, + nesting: event.data.nesting + 2, + tag: 'property', + attrs: { + __proto__: null, + name: 'flaky', + value: `${event.data.flakyRetriedCount} retries`, + }, + }, + ], }); } if (event.type === 'test:fail') { @@ -137,7 +211,11 @@ module.exports = async function* junitReporter(source) { __proto__: null, nesting: event.data.nesting + 1, tag: 'failure', - attrs: { __proto__: null, type: error?.failureType || error?.code, message: error?.message.trim() ?? '' }, + attrs: { + __proto__: null, + type: error?.failureType || error?.code, + message: error?.message.trim() ?? '', + }, children: [inspectWithNoCustomRetry(error, inspectOptions)], }); currentTest.failures = 1; @@ -149,10 +227,13 @@ module.exports = async function* junitReporter(source) { case 'test:diagnostic': { const parent = currentSuite?.children ?? roots; ArrayPrototypePush(parent, { - __proto__: null, nesting: event.data.nesting, comment: event.data.message, + __proto__: null, + nesting: event.data.nesting, + comment: event.data.message, }); break; - } default: + } + default: break; } } diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js index 595c96b48c8970..6578290f714ceb 100644 --- a/lib/internal/test_runner/reporter/tap.js +++ b/lib/internal/test_runner/reporter/tap.js @@ -19,7 +19,11 @@ const kDefaultIndent = ' '; // 4 spaces const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; const kDefaultTAPVersion = 13; -const inspectOptions = { __proto__: null, colors: false, breakLength: Infinity }; +const inspectOptions = { + __proto__: null, + colors: false, + breakLength: Infinity, +}; let testModule; // Lazy loaded due to circular dependency. function lazyLoadTest() { @@ -27,18 +31,38 @@ function lazyLoadTest() { return testModule; } - -async function * tapReporter(source) { +async function* tapReporter(source) { yield `TAP version ${kDefaultTAPVersion}\n`; for await (const { type, data } of source) { switch (type) { case 'test:fail': { - yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); - const location = data.file ? `${data.file}:${data.line}:${data.column}` : null; + yield reportTest( + data.nesting, + data.testNumber, + 'not ok', + data.name, + data.skip, + data.todo, + data.expectFailure, + data.flakyRetriedCount, + ); + const location = data.file + ? `${data.file}:${data.line}:${data.column}` + : null; yield reportDetails(data.nesting, data.details, location); break; - } case 'test:pass': - yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure, data.flaky); + } + case 'test:pass': + yield reportTest( + data.nesting, + data.testNumber, + 'ok', + data.name, + data.skip, + data.todo, + data.expectFailure, + data.flakyRetriedCount, + ); yield reportDetails(data.nesting, data.details, null); break; case 'test:plan': @@ -49,23 +73,42 @@ async function * tapReporter(source) { break; case 'test:stderr': case 'test:stdout': { - const lines = RegExpPrototypeSymbolSplit(kLineBreakRegExp, data.message); + const lines = RegExpPrototypeSymbolSplit( + kLineBreakRegExp, + data.message, + ); for (let i = 0; i < lines.length; i++) { if (lines[i].length === 0) continue; yield `# ${tapEscape(lines[i])}\n`; } break; - } case 'test:diagnostic': + } + case 'test:diagnostic': yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`; break; case 'test:coverage': - yield getCoverageReport(indent(data.nesting), data.summary, '# ', '', true); + yield getCoverageReport( + indent(data.nesting), + data.summary, + '# ', + '', + true, + ); break; } } } -function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure, flaky) { +function reportTest( + nesting, + testNumber, + status, + name, + skip, + todo, + expectFailure, + flakyRetriedCount, +) { let line = `${indent(nesting)}${status} ${testNumber}`; if (name) { @@ -78,10 +121,9 @@ function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } else if (expectFailure !== undefined) { line += ' # EXPECTED FAILURE'; - //should we use flaky >=0 here? for always printing 0 retries - } else if (flaky !== undefined && flaky > 0) { - const retryText = flaky === 1 ? 're-try' : 're-tries'; - line += ` # FLAKY ${flaky} ${retryText}`; + } else if (flakyRetriedCount !== undefined && flakyRetriedCount > 0) { + const retryText = flakyRetriedCount === 1 ? 're-try' : 're-tries'; + line += ` # FLAKY ${flakyRetriedCount} ${retryText}`; } line += '\n'; @@ -117,7 +159,6 @@ function indent(nesting) { return value; } - // In certain places, # and \ need to be escaped as \# and \\. function tapEscape(input) { let result = StringPrototypeReplaceAll(input, '\b', '\\b'); @@ -281,7 +322,12 @@ function jsToYaml(indent, name, value, seen) { } function isAssertionLike(value) { - return value && typeof value === 'object' && 'expected' in value && 'actual' in value; + return ( + value && + typeof value === 'object' && + 'expected' in value && + 'actual' in value + ); } module.exports = tapReporter; diff --git a/lib/internal/test_runner/reporter/utils.js b/lib/internal/test_runner/reporter/utils.js index 4f0f888572c2d1..1986adfb1572a8 100644 --- a/lib/internal/test_runner/reporter/utils.js +++ b/lib/internal/test_runner/reporter/utils.js @@ -17,7 +17,7 @@ const inspectOptions = { }; const reporterUnicodeSymbolMap = { - '__proto__': null, + __proto__: null, 'test:fail': '\u2716 ', 'test:pass': '\u2714 ', 'test:diagnostic': '\u2139 ', @@ -28,7 +28,7 @@ const reporterUnicodeSymbolMap = { }; const reporterColorMap = { - '__proto__': null, + __proto__: null, get 'test:fail'() { return colors.red; }, @@ -38,13 +38,13 @@ const reporterColorMap = { get 'test:diagnostic'() { return colors.blue; }, - get 'info'() { + get info() { return colors.blue; }, - get 'warn'() { + get warn() { return colors.yellow; }, - get 'error'() { + get error() { return colors.red; }, }; @@ -64,15 +64,25 @@ function formatError(error, indent) { RegExpPrototypeSymbolSplit( hardenRegExp(/\r?\n/), inspectWithNoCustomRetry(err, inspectOptions), - ), `\n${indent} `); + ), + `\n${indent} `, + ); return `\n${indent} ${message}\n`; } -function formatTestReport(type, data, showErrorDetails = true, prefix = '', indent = '') { +function formatTestReport( + type, + data, + showErrorDetails = true, + prefix = '', + indent = '', +) { let color = reporterColorMap[type] ?? colors.white; let symbol = reporterUnicodeSymbolMap[type] ?? ' '; - const { skip, todo, expectFailure, flaky } = data; - const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : ''; + const { skip, todo, expectFailure, flakyRetriedCount } = data; + const duration_ms = data.details?.duration_ms + ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` + : ''; let title = `${data.name}${duration_ms}`; if (skip !== undefined) { @@ -87,12 +97,15 @@ function formatTestReport(type, data, showErrorDetails = true, prefix = '', inde } } else if (expectFailure !== undefined) { title += ` # EXPECTED FAILURE`; - } else if (flaky !== undefined && flaky > 0) { - const retryText = flaky === 1 ? 're-try' : 're-tries'; - title += ` # FLAKY ${flaky} ${retryText}`; + } else if (flakyRetriedCount !== undefined && flakyRetriedCount > 0) { + const retryText = flakyRetriedCount === 1 ? 're-try' : 're-tries'; + title += ` # FLAKY ${flakyRetriedCount} ${retryText}`; } - const err = showErrorDetails && data.details?.error ? formatError(data.details.error, indent) : ''; + const err = + showErrorDetails && data.details?.error + ? formatError(data.details.error, indent) + : ''; return `${prefix}${indent}${color}${symbol}${title}${colors.white}${err}`; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index dffa7f0980d876..9b994ba7b714f8 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -12,6 +12,7 @@ const { FunctionPrototype, MathMax, Number, + NumberIsInteger, NumberPrototypeToFixed, ObjectSeal, Promise, @@ -31,17 +32,16 @@ const { SymbolDispose, } = primordials; const { getCallerLocation } = internalBinding('util'); -const { exitCodes: { kGenericUserError } } = internalBinding('errors'); +const { + exitCodes: { kGenericUserError }, +} = internalBinding('errors'); const { addAbortListener } = require('internal/events/abort_listener'); const { queueMicrotask } = require('internal/process/task_queues'); const { AsyncResource } = require('async_hooks'); const { AbortController } = require('internal/abort_controller'); const { AbortError, - codes: { - ERR_INVALID_ARG_TYPE, - ERR_TEST_FAILURE, - }, + codes: { ERR_INVALID_ARG_TYPE, ERR_TEST_FAILURE }, } = require('internal/errors'); const { MockTracker } = require('internal/test_runner/mock/mock'); const { TestsStream } = require('internal/test_runner/tests_stream'); @@ -65,10 +65,7 @@ const { validateOneOf, validateUint32, } = require('internal/validators'); -const { - clearTimeout, - setTimeout, -} = require('timers'); +const { clearTimeout, setTimeout } = require('timers'); const { TIMEOUT_MAX } = require('internal/timers'); const { fileURLToPath } = require('internal/url'); const { relative } = require('path'); @@ -90,8 +87,10 @@ const noop = FunctionPrototype; const kShouldAbort = Symbol('kShouldAbort'); const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']); const kUnwrapErrors = new SafeSet() - .add(kTestCodeFailure).add(kHookFailure) - .add('uncaughtException').add('unhandledRejection'); + .add(kTestCodeFailure) + .add(kHookFailure) + .add('uncaughtException') + .add('unhandledRejection'); let kResistStopPropagation; let assertObj; let findSourceMap; @@ -113,7 +112,9 @@ function lazyAssertObject(harness) { const { SnapshotManager } = require('internal/test_runner/snapshot'); assertObj = getAssertionMap(); - harness.snapshotManager = new SnapshotManager(harness.config.updateSnapshots); + harness.snapshotManager = new SnapshotManager( + harness.config.updateSnapshots, + ); if (!assertObj.has('snapshot')) { assertObj.set('snapshot', harness.snapshotManager.createAssert()); @@ -137,12 +138,16 @@ function stopTest(timeout, signal) { } else { timer = setTimeout(deferred.resolve, timeout); timer.unref(); - setOwnProperty(deferred, 'promise', PromisePrototypeThen(deferred.promise, () => { - throw new ERR_TEST_FAILURE( - `test timed out after ${timeout}ms`, - kTestTimeoutFailure, - ); - })); + setOwnProperty( + deferred, + 'promise', + PromisePrototypeThen(deferred.promise, () => { + throw new ERR_TEST_FAILURE( + `test timed out after ${timeout}ms`, + kTestTimeoutFailure, + ); + }), + ); disposeFunction = () => { abortListener[SymbolDispose](); @@ -155,15 +160,21 @@ function stopTest(timeout, signal) { } function testMatchesPattern(test, patterns) { - const matchesByNameOrParent = ArrayPrototypeSome(patterns, (re) => - RegExpPrototypeExec(re, test.name) !== null, - ) || (test.parent && testMatchesPattern(test.parent, patterns)); + const matchesByNameOrParent = + ArrayPrototypeSome( + patterns, + (re) => RegExpPrototypeExec(re, test.name) !== null, + ) || + (test.parent && testMatchesPattern(test.parent, patterns)); if (matchesByNameOrParent) return true; - const testNameWithAncestors = StringPrototypeTrim(test.getTestNameWithAncestors()); + const testNameWithAncestors = StringPrototypeTrim( + test.getTestNameWithAncestors(), + ); - return ArrayPrototypeSome(patterns, (re) => - RegExpPrototypeExec(re, testNameWithAncestors) !== null, + return ArrayPrototypeSome( + patterns, + (re) => RegExpPrototypeExec(re, testNameWithAncestors) !== null, ); } @@ -186,7 +197,11 @@ class TestPlan { validateNumber(wait, 'options.wait', 0, TIMEOUT_MAX); this.wait = wait; } else if (wait !== undefined) { - throw new ERR_INVALID_ARG_TYPE('options.wait', ['boolean', 'number'], wait); + throw new ERR_INVALID_ARG_TYPE( + 'options.wait', + ['boolean', 'number'], + wait, + ); } } @@ -249,7 +264,6 @@ class TestPlan { } } - class TestContext { #assert; #test; @@ -363,7 +377,11 @@ class TestContext { const subtest = this.#test.createSubtest( // eslint-disable-next-line no-use-before-define - Test, name, options, fn, overrides, + Test, + name, + options, + fn, + overrides, ); return subtest.start(); @@ -413,10 +431,7 @@ class TestContext { validateFunction(condition, 'condition'); validateObject(options, 'options'); - const { - interval = 50, - timeout = 1000, - } = options; + const { interval = 50, timeout = 1000 } = options; validateNumber(interval, 'options.interval', 0, TIMEOUT_MAX); validateNumber(timeout, 'options.timeout', 0, TIMEOUT_MAX); @@ -499,7 +514,19 @@ class Test extends AsyncResource { let { fn, name, parent } = options; - const { concurrency, entryFile, expectFailure, flaky, loc, only, timeout, todo, skip, signal, plan } = options; + const { + concurrency, + entryFile, + expectFailure, + flaky, + loc, + only, + timeout, + todo, + skip, + signal, + plan, + } = options; if (typeof fn !== 'function') { fn = noop; @@ -536,9 +563,10 @@ class Test extends AsyncResource { this.entryFile = entryFile; this.testDisambiguator = new SafeMap(); } else { - const nesting = parent.parent === null ? parent.nesting : - parent.nesting + 1; - const { config, isFilteringByName, isFilteringByOnly } = parent.root.harness; + const nesting = + parent.parent === null ? parent.nesting : parent.nesting + 1; + const { config, isFilteringByName, isFilteringByOnly } = + parent.root.harness; this.root = parent.root; this.harness = null; @@ -555,7 +583,11 @@ class Test extends AsyncResource { if (isFilteringByName) { this.filteredByName = this.willBeFilteredByName(); if (!this.filteredByName) { - for (let t = this.parent; t !== null && t.filteredByName; t = t.parent) { + for ( + let t = this.parent; + t !== null && t.filteredByName; + t = t.parent + ) { t.filteredByName = false; } } @@ -569,7 +601,11 @@ class Test extends AsyncResource { this.parent.runOnlySubtests = true; if (this.parent === this.root || this.parent.startTime === null) { - for (let t = this.parent; t !== null && !t.hasOnlyTests; t = t.parent) { + for ( + let t = this.parent; + t !== null && !t.hasOnlyTests; + t = t.parent + ) { t.hasOnlyTests = true; } } @@ -591,8 +627,8 @@ class Test extends AsyncResource { case 'boolean': if (concurrency) { - this.concurrency = parent === null ? - MathMax(availableParallelism() - 1, 1) : Infinity; + this.concurrency = + parent === null ? MathMax(availableParallelism() - 1, 1) : Infinity; } else { this.concurrency = 1; } @@ -600,7 +636,11 @@ class Test extends AsyncResource { default: if (concurrency != null) - throw new ERR_INVALID_ARG_TYPE('options.concurrency', ['boolean', 'number'], concurrency); + throw new ERR_INVALID_ARG_TYPE( + 'options.concurrency', + ['boolean', 'number'], + concurrency, + ); } if (timeout != null && timeout !== Infinity) { @@ -624,14 +664,14 @@ class Test extends AsyncResource { validateAbortSignal(signal, 'options.signal'); if (signal) { - kResistStopPropagation ??= require('internal/event_target').kResistStopPropagation; + kResistStopPropagation ??= + require('internal/event_target').kResistStopPropagation; } - this.outerSignal?.addEventListener( - 'abort', - this.#abortHandler, - { __proto__: null, [kResistStopPropagation]: true }, - ); + this.outerSignal?.addEventListener('abort', this.#abortHandler, { + __proto__: null, + [kResistStopPropagation]: true, + }); this.fn = fn; this.mock = null; @@ -639,6 +679,24 @@ class Test extends AsyncResource { this.expectedAssertions = plan; this.cancelled = false; this.expectFailure = expectFailure !== undefined && expectFailure !== false; + + // Validate and process flaky option + if (flaky != null && flaky !== false) { + this.flakyRetries = flaky === true ? kDefaultFlakyRetries : flaky; + if (!NumberIsInteger(this.flakyRetries) || this.flakyRetries < 1) { + throw new ERR_INVALID_ARG_TYPE( + 'options.flaky', + ['boolean', 'positive integer'], + flaky, + ); + } + } else if (flaky === undefined && this.parent?.flakyRetries > 0) { + // Inherit flaky from parent suite + this.flakyRetries = this.parent.flakyRetries; + } else { + this.flakyRetries = 0; + } + this.retriesTaken = 0; this.skipped = skip !== undefined && skip !== false; this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo; this.startTime = null; @@ -647,8 +705,8 @@ class Test extends AsyncResource { this.error = null; this.attempt = undefined; this.passedAttempt = undefined; - this.message = typeof skip === 'string' ? skip : - typeof todo === 'string' ? todo : null; + this.message = + typeof skip === 'string' ? skip : typeof todo === 'string' ? todo : null; this.activeSubtests = 0; this.pendingSubtests = []; this.readySubtests = new SafeMap(); @@ -702,16 +760,24 @@ class Test extends AsyncResource { this.root.testDisambiguator.set(testIdentifier, 1); } this.attempt = this.root.harness.previousRuns.length; - const previousAttempt = this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]; + const previousAttempt = + this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]; if (previousAttempt != null) { this.passedAttempt = previousAttempt.passed_on_attempt; this.fn = () => { for (let i = 0; i < (previousAttempt.children?.length ?? 0); i++) { const child = previousAttempt.children[i]; - this.createSubtest(Test, child.name, { __proto__: null }, noop, { - __proto__: null, - loc: [child.line, child.column, child.file], - }, noop).start(); + this.createSubtest( + Test, + child.name, + { __proto__: null }, + noop, + { + __proto__: null, + loc: [child.line, child.column, child.file], + }, + noop, + ).start(); } }; } @@ -729,10 +795,16 @@ class Test extends AsyncResource { return; } - if (this.root.harness.isFilteringByOnly && !this.only && !this.hasOnlyTests) { - if (this.parent.runOnlySubtests || - this.parent.hasOnlyTests || - this.only === false) { + if ( + this.root.harness.isFilteringByOnly && + !this.only && + !this.hasOnlyTests + ) { + if ( + this.parent.runOnlySubtests || + this.parent.hasOnlyTests || + this.only === false + ) { this.filtered = true; } } @@ -785,7 +857,12 @@ class Test extends AsyncResource { while (this.pendingSubtests.length > 0 && this.hasConcurrency()) { const deferred = ArrayPrototypeShift(this.pendingSubtests); const test = deferred.test; - test.reporter.dequeue(test.nesting, test.loc, test.name, this.reportedType); + test.reporter.dequeue( + test.nesting, + test.loc, + test.name, + this.reportedType, + ); await test.run(); deferred.resolve(); } @@ -798,8 +875,10 @@ class Test extends AsyncResource { addReadySubtest(subtest) { this.readySubtests.set(subtest.childNumber, subtest); - if (this.unfinishedSubtests.delete(subtest) && - this.unfinishedSubtests.size === 0) { + if ( + this.unfinishedSubtests.delete(subtest) && + this.unfinishedSubtests.size === 0 + ) { this.subtestsPromise.resolve(); } } @@ -863,7 +942,14 @@ class Test extends AsyncResource { } } - const test = new Factory({ __proto__: null, fn, name, parent, ...options, ...overrides }); + const test = new Factory({ + __proto__: null, + fn, + name, + parent, + ...options, + ...overrides, + }); if (parent.waitingOn === 0) { parent.waitingOn = test.childNumber; @@ -884,7 +970,8 @@ class Test extends AsyncResource { } #abortHandler = () => { - const error = this.outerSignal?.reason || new AbortError('The test was aborted'); + const error = + this.outerSignal?.reason || new AbortError('The test was aborted'); error.failureType = kAborted; this.#cancel(error); }; @@ -894,11 +981,12 @@ class Test extends AsyncResource { return; } - this.fail(error || - new ERR_TEST_FAILURE( - 'test did not finish before its parent and was cancelled', - kCancelledByParent, - ), + this.fail( + error || + new ERR_TEST_FAILURE( + 'test did not finish before its parent and was cancelled', + kCancelledByParent, + ), ); this.cancelled = true; this.abortController.abort(); @@ -914,7 +1002,8 @@ class Test extends AsyncResource { if (this.parent.hooks.afterEach.length > 0) { ArrayPrototypePushApply( - this.hooks.afterEach, ArrayPrototypeSlice(this.parent.hooks.afterEach), + this.hooks.afterEach, + ArrayPrototypeSlice(this.parent.hooks.afterEach), ); } } @@ -938,7 +1027,12 @@ class Test extends AsyncResource { // afterEach hooks for the current test should run in the order that they // are created. However, the current test's afterEach hooks should run // prior to any ancestor afterEach hooks. - ArrayPrototypeSplice(this.hooks[name], this.hooks.ownAfterEachCount, 0, hook); + ArrayPrototypeSplice( + this.hooks[name], + this.hooks.ownAfterEachCount, + 0, + hook, + ); this.hooks.ownAfterEachCount++; } else { ArrayPrototypePush(this.hooks[name], hook); @@ -1041,7 +1135,10 @@ class Test extends AsyncResource { } } } catch (err) { - const error = new ERR_TEST_FAILURE(`failed running ${hook} hook`, kHookFailure); + const error = new ERR_TEST_FAILURE( + `failed running ${hook} hook`, + kHookFailure, + ); error.cause = isTestFailureError(err) ? err.cause : err; throw error; } @@ -1095,53 +1192,69 @@ class Test extends AsyncResource { await this.parent.runHook('beforeEach', hookArgs); } stopPromise = stopTest(this.timeout, this.signal); - const runArgs = ArrayPrototypeSlice(args); - ArrayPrototypeUnshift(runArgs, this.fn, ctx); - const promises = []; - if (this.fn.length === runArgs.length - 1) { - // This test is using legacy Node.js error-first callbacks. - const { promise, cb } = createDeferredCallback(); - ArrayPrototypePush(runArgs, cb); - - const ret = ReflectApply(this.runInAsyncScope, this, runArgs); - - if (isPromise(ret)) { - this.fail(new ERR_TEST_FAILURE( - 'passed a callback but also returned a Promise', - kCallbackAndPromisePresent, - )); - ArrayPrototypePush(promises, ret); - } else { - ArrayPrototypePush(promises, PromiseResolve(promise)); + const maxAttempts = this.flakyRetries > 0 ? this.flakyRetries + 1 : 1; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + if (attempt > 1) { + this.error = null; + this.passed = false; } - } else { - // This test is synchronous or using Promises. - const promise = ReflectApply(this.runInAsyncScope, this, runArgs); - ArrayPrototypePush(promises, PromiseResolve(promise)); - } - ArrayPrototypePush(promises, stopPromise); + try { + const runArgs = ArrayPrototypeSlice(args); + ArrayPrototypeUnshift(runArgs, this.fn, ctx); + + const promises = []; + if (this.fn.length === runArgs.length - 1) { + const { promise, cb } = createDeferredCallback(); + ArrayPrototypePush(runArgs, cb); + + const ret = ReflectApply(this.runInAsyncScope, this, runArgs); + + if (isPromise(ret)) { + this.fail( + new ERR_TEST_FAILURE( + 'passed a callback but also returned a Promise', + kCallbackAndPromisePresent, + ), + ); + ArrayPrototypePush(promises, ret); + } else { + ArrayPrototypePush(promises, PromiseResolve(promise)); + } + } else { + const promise = ReflectApply(this.runInAsyncScope, this, runArgs); + ArrayPrototypePush(promises, PromiseResolve(promise)); + } - // Wait for the race to finish - await SafePromiseRace(promises); + ArrayPrototypePush(promises, stopPromise); - this[kShouldAbort](); + await SafePromiseRace(promises); - if (this.subtestsPromise !== null) { - await SafePromiseRace([this.subtestsPromise.promise, stopPromise]); - } + this[kShouldAbort](); + + if (this.subtestsPromise !== null) { + await SafePromiseRace([this.subtestsPromise.promise, stopPromise]); + } + + if (this.plan !== null) { + const planPromise = this.plan?.check(); + if (planPromise) { + await SafePromiseRace([planPromise, stopPromise]); + } + } - if (this.plan !== null) { - const planPromise = this.plan?.check(); - // If the plan returns a promise, it means that it is waiting for more assertions to be made before - // continuing. - if (planPromise) { - await SafePromiseRace([planPromise, stopPromise]); + this.pass(); + break; + } catch (attemptErr) { + if (attempt < maxAttempts) { + this.retriesTaken = attempt; + continue; + } + throw attemptErr; } } - - this.pass(); await afterEach(); await after(); } catch (err) { @@ -1154,8 +1267,16 @@ class Test extends AsyncResource { } else { this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure)); } - try { await afterEach(); } catch { /* test is already failing, let's ignore the error */ } - try { await after(); } catch { /* Ignore error. */ } + try { + await afterEach(); + } catch { + /* test is already failing, let's ignore the error */ + } + try { + await after(); + } catch { + /* Ignore error. */ + } } finally { stopPromise?.[SymbolDispose](); @@ -1186,15 +1307,21 @@ class Test extends AsyncResource { for (let i = 0; i < reporterScope.reporters.length; i++) { const { destination } = reporterScope.reporters[i]; - ArrayPrototypePush(promises, new Promise((resolve) => { - destination.on('unpipe', () => { - if (!destination.closed && typeof destination.close === 'function') { - destination.close(resolve); - } else { - resolve(); - } - }); - })); + ArrayPrototypePush( + promises, + new Promise((resolve) => { + destination.on('unpipe', () => { + if ( + !destination.closed && + typeof destination.close === 'function' + ) { + destination.close(resolve); + } else { + resolve(); + } + }); + }), + ); } this.harness.teardown(); @@ -1240,7 +1367,14 @@ class Test extends AsyncResource { const report = this.getReportDetails(); report.details.passed = this.passed; this.testNumber ||= ++this.parent.outputSubtestCount; - this.reporter.complete(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.reporter.complete( + this.nesting, + this.loc, + this.testNumber, + this.name, + report.details, + report.directive, + ); this.parent.activeSubtests--; } @@ -1248,13 +1382,7 @@ class Test extends AsyncResource { this.parent.processReadySubtestRange(false); this.parent.processPendingSubtests(); } else if (!this.reported) { - const { - diagnostics, - harness, - loc, - nesting, - reporter, - } = this; + const { diagnostics, harness, loc, nesting, reporter } = this; this.reported = true; reporter.plan(nesting, loc, harness.counters.topLevel); @@ -1271,21 +1399,38 @@ class Test extends AsyncResource { reporter.diagnostic(nesting, loc, `suites ${harness.counters.suites}`); reporter.diagnostic(nesting, loc, `pass ${harness.counters.passed}`); reporter.diagnostic(nesting, loc, `fail ${harness.counters.failed}`); - reporter.diagnostic(nesting, loc, `cancelled ${harness.counters.cancelled}`); + reporter.diagnostic( + nesting, + loc, + `cancelled ${harness.counters.cancelled}`, + ); reporter.diagnostic(nesting, loc, `skipped ${harness.counters.skipped}`); reporter.diagnostic(nesting, loc, `todo ${harness.counters.todo}`); + reporter.diagnostic(nesting, loc, `flaky ${harness.counters.flaky}`); reporter.diagnostic(nesting, loc, `duration_ms ${duration}`); if (coverage) { const coverages = [ - { __proto__: null, actual: coverage.totals.coveredLinePercent, - threshold: this.config.lineCoverage, name: 'line' }, - - { __proto__: null, actual: coverage.totals.coveredBranchPercent, - threshold: this.config.branchCoverage, name: 'branch' }, - - { __proto__: null, actual: coverage.totals.coveredFunctionPercent, - threshold: this.config.functionCoverage, name: 'function' }, + { + __proto__: null, + actual: coverage.totals.coveredLinePercent, + threshold: this.config.lineCoverage, + name: 'line', + }, + + { + __proto__: null, + actual: coverage.totals.coveredBranchPercent, + threshold: this.config.branchCoverage, + name: 'branch', + }, + + { + __proto__: null, + actual: coverage.totals.coveredFunctionPercent, + threshold: this.config.functionCoverage, + name: 'function', + }, ]; for (let i = 0; i < coverages.length; i++) { @@ -1293,7 +1438,12 @@ class Test extends AsyncResource { if (actual < threshold) { harness.success = false; process.exitCode = kGenericUserError; - reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`, 'error'); + reporter.diagnostic( + nesting, + loc, + `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`, + 'error', + ); } } @@ -1301,7 +1451,11 @@ class Test extends AsyncResource { } reporter.summary( - nesting, loc?.file, harness.success, harness.counters, duration, + nesting, + loc?.file, + harness.success, + harness.counters, + duration, ); if (harness.watching) { @@ -1315,10 +1469,11 @@ class Test extends AsyncResource { } isClearToSend() { - return this.parent === null || - ( - this.parent.waitingOn === this.childNumber && this.parent.isClearToSend() - ); + return ( + this.parent === null || + (this.parent.waitingOn === this.childNumber && + this.parent.isClearToSend()) + ); } finalize() { @@ -1337,8 +1492,10 @@ class Test extends AsyncResource { this.parent.waitingOn++; this.finished = true; - if (this.parent === this.root && - this.root.waitingOn > this.root.subtests.length) { + if ( + this.parent === this.root && + this.root.waitingOn > this.root.subtests.length + ) { // At this point all of the tests have finished running. However, there // might be ref'ed handles keeping the event loop alive. This gives the // global after() hook a chance to clean them up. The user may also @@ -1362,6 +1519,8 @@ class Test extends AsyncResource { directive = this.reporter.getTodo(this.message); } else if (this.expectFailure) { directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure + } else if (this.flakyRetries > 0 && this.retriesTaken > 0 && this.passed) { + directive = this.reporter.getFlaky(this.retriesTaken); } if (this.reportedType) { @@ -1383,16 +1542,34 @@ class Test extends AsyncResource { report() { countCompletedTest(this); if (this.outputSubtestCount > 0) { - this.reporter.plan(this.subtests[0].nesting, this.loc, this.outputSubtestCount); + this.reporter.plan( + this.subtests[0].nesting, + this.loc, + this.outputSubtestCount, + ); } else { this.reportStarted(); } const report = this.getReportDetails(); if (this.passed) { - this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.reporter.ok( + this.nesting, + this.loc, + this.testNumber, + this.name, + report.details, + report.directive, + ); } else { - this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive); + this.reporter.fail( + this.nesting, + this.loc, + this.testNumber, + this.name, + report.details, + report.directive, + ); } for (let i = 0; i < this.diagnostics.length; i++) { @@ -1459,11 +1636,18 @@ class TestHook extends Test { } this.endTime ??= hrtime(); - parent.reporter.fail(0, loc, parent.subtests.length + 1, loc.file, { - __proto__: null, - duration_ms: this.duration(), - error, - }, undefined); + parent.reporter.fail( + 0, + loc, + parent.subtests.length + 1, + loc.file, + { + __proto__: null, + duration_ms: this.duration(), + error, + }, + undefined, + ); } } } @@ -1476,9 +1660,11 @@ class Suite extends Test { this.timeout = null; } - if (this.config.testNamePatterns !== null && - this.config.testSkipPatterns !== null && - !options.skip) { + if ( + this.config.testNamePatterns !== null && + this.config.testSkipPatterns !== null && + !options.skip + ) { this.fn = options.fn || this.fn; this.skipped = false; } @@ -1510,7 +1696,10 @@ class Suite extends Test { const hookArgs = this.getRunArgs(); let stopPromise; - const after = runOnce(() => this.runHook('after', hookArgs), kRunOnceOptions); + const after = runOnce( + () => this.runHook('after', hookArgs), + kRunOnceOptions, + ); try { this.parent.activeSubtests++; await this.buildSuite; @@ -1537,7 +1726,11 @@ class Suite extends Test { this.pass(); } catch (err) { - try { await after(); } catch { /* suite is already failing */ } + try { + await after(); + } catch { + /* suite is already failing */ + } if (isTestFailureError(err)) { this.fail(err); } else { diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js index 7b64487696f53f..5e5ed10044165e 100644 --- a/lib/internal/test_runner/tests_stream.js +++ b/lib/internal/test_runner/tests_stream.js @@ -91,6 +91,10 @@ class TestsStream extends Readable { return { __proto__: null, expectFailure: expectation ?? true }; } + getFlaky(retriesTaken = undefined) { + return { __proto__: null, flakyRetriedCount: retriesTaken ?? 0 }; + } + enqueue(nesting, loc, name, type) { this[kEmitMessage]('test:enqueue', { __proto__: null, diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 93bc0662b7b834..00000000000000 --- a/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "node", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/test/fixtures/test-runner/output/flaky.js b/test/fixtures/test-runner/output/flaky.js new file mode 100644 index 00000000000000..69a8973159d5e0 --- /dev/null +++ b/test/fixtures/test-runner/output/flaky.js @@ -0,0 +1,68 @@ +// Flags: --test-reporter=tap +'use strict'; +require('../../../common'); +const { test, describe, it } = require('node:test'); + +// Track invocation count for flaky tests +let flakyTestInvocations = 0; +let flakyTestMethodInvocations = 0; +let flakyTestAlwaysFails = 0; + +// Test that passes after 2 retries (3rd attempt succeeds) +test('flaky test that passes on retry', { flaky: 5 }, () => { + flakyTestInvocations++; + if (flakyTestInvocations < 3) { + throw new Error('Simulated failure'); + } + // Passes on 3rd attempt +}); + +// Test using method syntax +it.flaky('flaky test using method syntax', () => { + flakyTestMethodInvocations++; + if (flakyTestMethodInvocations < 2) { + throw new Error('Simulated failure'); + } + // Passes on 2nd attempt +}); + +// Test that always fails (exhausts retries) +test('flaky test that always fails', { flaky: 3 }, () => { + flakyTestAlwaysFails++; + throw new Error(`Always fails (attempt ${flakyTestAlwaysFails})`); +}); + +// Test with flaky: true (should use default 20) +test('flaky test with true value', { flaky: true }, () => { + // This one passes immediately +}); + +// Suite with flaky option +describe.flaky('flaky suite', () => { + it('inherits flaky from suite', () => { + // Passes immediately + }); + + it('override flaky in test', { flaky: false }, () => { + // Not flaky despite suite being flaky + }); +}); + +// Invalid flaky values should error +try { + test('invalid flaky value -1', { flaky: -1 }, () => {}); +} catch (err) { + console.log('Expected error for negative flaky:', err.message); +} + +try { + test('invalid flaky value 0', { flaky: 0 }, () => {}); +} catch (err) { + console.log('Expected error for zero flaky:', err.message); +} + +try { + test('invalid flaky value string', { flaky: 'invalid' }, () => {}); +} catch (err) { + console.log('Expected error for string flaky:', err.message); +} diff --git a/test/fixtures/test-runner/output/flaky.snapshot b/test/fixtures/test-runner/output/flaky.snapshot new file mode 100644 index 00000000000000..5d2b8aeaf14849 --- /dev/null +++ b/test/fixtures/test-runner/output/flaky.snapshot @@ -0,0 +1,69 @@ +TAP version 13 +# Subtest: flaky test that passes on retry +ok 1 - flaky test that passes on retry # FLAKY 2 re-tries + --- + duration_ms: * + type: 'test' + ... +# Subtest: flaky test using method syntax +ok 2 - flaky test using method syntax # FLAKY 1 re-try + --- + duration_ms: * + type: 'test' + ... +# Subtest: flaky test that always fails +not ok 3 - flaky test that always fails + --- + duration_ms: * + type: 'test' + location: '/test/fixtures/test-runner/output/flaky.js:(LINE):1' + failureType: 'testCodeFailure' + error: 'Always fails (attempt 4)' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: flaky test with true value +ok 4 - flaky test with true value + --- + duration_ms: * + type: 'test' + ... +# Subtest: flaky suite + # Subtest: inherits flaky from suite + ok 1 - inherits flaky from suite + --- + duration_ms: * + type: 'test' + ... + # Subtest: override flaky in test + ok 2 - override flaky in test + --- + duration_ms: * + type: 'test' + ... + 1..2 +ok 5 - flaky suite + --- + duration_ms: * + type: 'suite' + ... +Expected error for negative flaky: * +Expected error for zero flaky: * +Expected error for string flaky: * +1..5 +# tests 6 +# suites 1 +# pass 5 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# flaky 5 +# duration_ms * diff --git a/test/test-runner/test-output-flaky.mjs b/test/test-runner/test-output-flaky.mjs new file mode 100644 index 00000000000000..55ac25c8dde232 --- /dev/null +++ b/test/test-runner/test-output-flaky.mjs @@ -0,0 +1,11 @@ +// Test that the output of test-runner/output/flaky.js matches test-runner/output/flaky.snapshot +import '../common/index.mjs'; +import * as fixtures from '../common/fixtures.mjs'; +import { spawnAndAssert, defaultTransform, ensureCwdIsProjectRoot } from '../common/assertSnapshot.js'; + +ensureCwdIsProjectRoot(); +await spawnAndAssert( + fixtures.path('test-runner/output/flaky.js'), + defaultTransform, + { flags: ['--test-reporter=tap'] }, +);