test_runner: fix passing expectFailure

Author: MoLow

lib/internal/test_runner/test.js

@@ -82,6 +82,7 @@ const kParentAlreadyFinished = 'parentAlreadyFinished';
 const kSubtestsFailed = 'subtestsFailed';
 const kTestCodeFailure = 'testCodeFailure';
 const kTestTimeoutFailure = 'testTimeoutFailure';
+const kExpectedFailure = 'expectedFailure';
 const kHookFailure = 'hookFailed';
 const kDefaultTimeout = null;
 const noop = FunctionPrototype;
@@ -957,6 +958,14 @@ class Test extends AsyncResource {
   }
 
   pass() {
+    if (this.error == null && this.expectFailure === true) {
+      this.passed = false;
+      this.error = new ERR_TEST_FAILURE(
+        'test was expected to fail but passed',
+        kExpectedFailure,
+      );
+      return;
+    }
     if (this.error !== null) {
       return;
     }

test/fixtures/test-runner/output/describe_it.snapshot

@@ -84,10 +84,14 @@ ok 10 - sync skip pass # SKIP
   type: 'test'
   ...
 # Subtest: sync skip expect fail
-ok 11 - sync skip expect fail # SKIP
+not ok 11 - sync skip expect fail # SKIP
   ---
   duration_ms: *
   type: 'test'
+  location: '/test/fixtures/test-runner/output/describe_it.js:(LINE):4'
+  failureType: 'expectedFailure'
+  error: 'test was expected to fail but passed'
+  code: 'ERR_TEST_FAILURE'
   ...
 # Subtest: sync skip pass with message
 ok 12 - sync skip pass with message # SKIP this is skipped

test/fixtures/test-runner/output/dot_reporter.snapshot

@@ -1,11 +1,13 @@
-........XX...X..XXX.
+.....X..XX...X..XXX.
 X.....XXX...........
 .X.........X...XXX.X
 X.....XXXXXXX...XXXX
 X
 
 Failed tests:
 
+﹣ sync skip expect fail (*ms) # SKIP
+  'test was expected to fail but passed'
 ✖ sync fail todo (*ms) # TODO
   Error: thrown from sync fail todo
       *

test/fixtures/test-runner/output/junit_reporter.snapshot

@@ -7,8 +7,11 @@
 	<testcase name="sync todo with expect fail" time="*" classname="test" file="*">
 		<skipped type="todo" message="true"/>
 	</testcase>
-	<testcase name="sync skip expect fail" time="*" classname="test" file="*">
+	<testcase name="sync skip expect fail" time="*" classname="test" file="*" failure="test was expected to fail but passed">
 		<skipped type="skipped" message="true"/>
+		<failure type="expectedFailure" message="test was expected to fail but passed">
+[Error [ERR_TEST_FAILURE]: test was expected to fail but passed] { code: 'ERR_TEST_FAILURE', failureType: 'expectedFailure', cause: 'test was expected to fail but passed' }
+		</failure>
 	</testcase>
 	<testcase name="sync pass todo" time="*" classname="test" file="*">
 		<skipped type="todo" message="true"/>

test/fixtures/test-runner/output/output.snapshot

@@ -30,10 +30,14 @@ ok 5 - sync todo with expect fail # TODO
   type: 'test'
   ...
 # Subtest: sync skip expect fail
-ok 6 - sync skip expect fail # SKIP
+not ok 6 - sync skip expect fail # SKIP
   ---
   duration_ms: *
   type: 'test'
+  location: '/test/fixtures/test-runner/output/output.js:(LINE):6'
+  failureType: 'expectedFailure'
+  error: 'test was expected to fail but passed'
+  code: 'ERR_TEST_FAILURE'
   ...
 # Subtest: sync pass todo
 ok 7 - sync pass todo # TODO

test/fixtures/test-runner/output/output_cli.snapshot

@@ -30,10 +30,14 @@ ok 5 - sync todo with expect fail # TODO
   type: 'test'
   ...
 # Subtest: sync skip expect fail
-ok 6 - sync skip expect fail # SKIP
+not ok 6 - sync skip expect fail # SKIP
   ---
   duration_ms: *
   type: 'test'
+  location: '/test/fixtures/test-runner/output/output.js:(LINE):6'
+  failureType: 'expectedFailure'
+  error: 'test was expected to fail but passed'
+  code: 'ERR_TEST_FAILURE'
   ...
 # Subtest: sync pass todo
 ok 7 - sync pass todo # TODO

test/fixtures/test-runner/output/spec_reporter.snapshot

@@ -105,6 +105,10 @@
 
  failing tests:
 
+*
+ sync skip expect fail (*ms) # SKIP
+  'test was expected to fail but passed'
+
 *
  sync fail todo (*ms) # TODO
   Error: thrown from sync fail todo

test/fixtures/test-runner/output/spec_reporter_cli.snapshot

@@ -108,6 +108,10 @@
 
  failing tests:
 
+*
+ sync skip expect fail (*ms) # SKIP
+  'test was expected to fail but passed'
+
 *
  sync fail todo (*ms) # TODO
   Error: thrown from sync fail todo

test/parallel/test-runner-expect-error-but-pass.js

@@ -0,0 +1,18 @@
+'use strict';
+const common = require('../common');
+const assert = require('node:assert');
+const { run, test } = require('node:test');
+
+if (!process.env.NODE_TEST_CONTEXT) {
+  const stream = run({ files: [__filename] });
+
+  stream.on('test:pass', common.mustNotCall());
+  stream.on('test:fail', common.mustCall((event) => {
+    assert.strictEqual(event.expectFailure, true);
+    assert.strictEqual(event.details.error.code, 'ERR_TEST_FAILURE');
+    assert.strictEqual(event.details.error.failureType, 'expectedFailure');
+    assert.strictEqual(event.details.error.cause, 'test was expected to fail but passed');
+  }, 1));
+} else {
+  test('passing test', { expectFailure: true }, () => {});
+}

test/parallel/test-runner-expect-error.js

@@ -0,0 +1,15 @@
+'use strict';
+const common = require('../common');
+const assert = require('node:assert');
+const { run, test } = require('node:test');
+
+if (!process.env.NODE_TEST_CONTEXT) {
+  const stream = run({ files: [__filename] });
+
+  stream.on('test:fail', common.mustNotCall());
+  stream.on('test:pass', common.mustCall((event) => {
+    assert.strictEqual(event.expectFailure, true);
+  }, 1));
+} else {
+  test('failing test', { expectFailure: true }, () => assert.fail('should not pass'));
+}