[WIP] A way to submit tests that should fail.

lib/runner.js

@@ -16,6 +16,7 @@ var chainableMethods = {
 		exclusive: false,
 		skipped: false,
 		todo: false,
+		failing: false,
 		callback: false,
 		always: false
 	},
@@ -26,6 +27,7 @@ var chainableMethods = {
 		after: {type: 'after'},
 		skip: {skipped: true},
 		todo: {todo: true},
+		failing: {failing: true},
 		only: {exclusive: true},
 		beforeEach: {type: 'beforeEach'},
 		afterEach: {type: 'afterEach'},

lib/test.js

@@ -186,8 +186,15 @@ Test.prototype.run = function () {
 };
 
 Test.prototype._result = function () {
-	var passed = this.assertError === undefined;
-	return {passed: passed, result: this, reason: this.assertError};
+	var reason = this.assertError;
+	var passed = reason === undefined;
+	if (this.metadata.failing) {
+		passed = !passed;
+		if (!passed) {
+			reason = new Error('Test was expected to fail, but succeeded, you should unmark the test as failing');
+		}
+	}
+	return {passed: passed, result: this, reason: reason};
 };
 
 Object.defineProperty(Test.prototype, 'end', {

readme.md

@@ -415,6 +415,19 @@ You can use the `.todo` modifier when you're planning to write a test. Like skip
 test.todo('will think about writing this later');
 ```
 
+### Failing tests
+
+You can use the `.failing` modifier to document issues with your code that need to be fixed. Failing tests are run just like normal ones, but they are expected to fail, and will not break your build when they do. If a test marked as failing actually passes, it will be reported as an error and fail the build with a helpful message instructing you to remove the .failing modifier.
+
+This allows you to merge `.failing` tests before a fix is implemented without breaking CI. This is a great way to recognize good bug report PR's with a commit credit, even if the reporter is unable to actually fix the problem.
+
+```js
+// See: github.com/user/repo/issues/1234
+test.failing('demonstrate some bug', t => {
+	t.fail(); // test will count as passed
+});
+```
+
 ### Before & after hooks
 
 AVA lets you register hooks that are run before and after your tests. This allows you to run setup and/or teardown code.

test/test.js

@@ -5,18 +5,32 @@ var delay = require('delay');
 var isPromise = require('is-promise');
 var Test = require('../lib/test');
 
+var failingTestHint = 'Test was expected to fail, but succeeded, you should unmark the test as failing';
+
 function ava(title, fn, contextRef, report) {
 	var t = new Test(title, fn, contextRef, report);
 	t.metadata = {callback: false};
 	return t;
 }
 
+ava.failing = function (title, fn, contextRef, report) {
+	var t = new Test(title, fn, contextRef, report);
+	t.metadata = {callback: false, failing: true};
+	return t;
+};
+
 ava.cb = function (title, fn, contextRef, report) {
 	var t = new Test(title, fn, contextRef, report);
 	t.metadata = {callback: true};
 	return t;
 };
 
+ava.cb.failing = function (title, fn, contextRef, report) {
+	var t = new Test(title, fn, contextRef, report);
+	t.metadata = {callback: true, failing: true};
+	return t;
+};
+
 test('must be called with new', function (t) {
 	t.throws(function () {
 		var test = Test;
@@ -34,6 +48,25 @@ test('run test', function (t) {
 	t.end();
 });
 
+test('expected failing test', function (t) {
+	var result = ava.failing('foo', function (a) {
+		a.fail();
+	}).run();
+
+	t.is(result.passed, true);
+	t.end();
+});
+
+test('fail a failing test if it pass', function (t) {
+	var result = ava.failing('foo', function (a) {
+		a.pass();
+	}).run();
+
+	t.is(result.passed, false);
+	t.is(result.reason.message, failingTestHint);
+	t.end();
+});
+
 test('title is optional', function (t) {
 	var result = ava(function (a) {
 		a.pass();
@@ -138,6 +171,27 @@ test('end can be used as callback with error', function (t) {
 	});
 });
 
+test('fail a failing callback test if it passed', function (t) {
+	ava.cb.failing(function (a) {
+		a.end();
+	}).run().then(function (result) {
+		t.is(result.passed, false);
+		t.is(result.reason.message, failingTestHint);
+		t.end();
+	});
+});
+
+test('failing can work with callback', function (t) {
+	var err = new Error('failed');
+	ava.cb.failing(function (a) {
+		a.end(err);
+	}).run().then(function (result) {
+		t.is(result.passed, true);
+		t.is(result.reason, err);
+		t.end();
+	});
+});
+
 test('end can be used as callback with a non-error as its error argument', function (t) {
 	var nonError = {foo: 'bar'};
 	ava.cb(function (a) {
@@ -587,3 +641,45 @@ test('it is an error to set context in a hook', function (t) {
 	t.match(result.reason.message, /t\.context is not available in foo tests/);
 	t.end();
 });
+
+test('failing test returns a resolved promise is failure', function (t) {
+	ava.failing(function (a) {
+		a.plan(1);
+		a.notThrows(delay(10), 'foo');
+		return Promise.resolve();
+	}).run().then(function (result) {
+		t.is(result.passed, false);
+		t.is(result.reason.message, failingTestHint);
+		t.end();
+	});
+});
+
+test('failing test returns a rejected promise is passing', function (t) {
+	ava.failing(function (a) {
+		a.plan(1);
+		a.notThrows(delay(10), 'foo');
+		return Promise.reject();
+	}).run().then(function (result) {
+		t.is(result.passed, true);
+		t.end();
+	});
+});
+
+test('failing test with t.throws(nonThrowingPromise) is passing', function (t) {
+	ava.failing(function (a) {
+		a.throws(Promise.resolve(10));
+	}).run().then(function (result) {
+		t.is(result.passed, true);
+		t.end();
+	});
+});
+
+test('failing test with t.notThrows(throws) is failure', function (t) {
+	ava.failing(function (a) {
+		a.notThrows(Promise.resolve('foo'));
+	}).run().then(function (result) {
+		t.is(result.passed, false);
+		t.is(result.reason.message, failingTestHint);
+		t.end();
+	});
+});