Skip to content

Commit 8d4606e

Browse files
AriPerkkioAriPerkkio
authored
feat(watch): test run cancelling, feat: --bail option for cancelling test run (#3163)
1 parent 97b1b4a commit 8d4606e

File tree

35 files changed

+433
-33
lines changed

35 files changed

+433
-33
lines changed

‎docs/advanced/runner.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ export interface VitestRunner {
1717
*/
1818
onCollected?(files: File[]): unknown
1919

20+
/**
21+
* Called when test runner should cancel next test runs.
22+
* Runner should listen for this method and mark tests and suites as skipped in
23+
* "onBeforeRunSuite" and "onBeforeRunTest" when called.
24+
*/
25+
onCancel?(reason: CancelReason): unknown
26+
2027
/**
2128
* Called before running a single test. Doesn't have "result" yet.
2229
*/
@@ -86,7 +93,7 @@ export interface VitestRunner {
8693
When initiating this class, Vitest passes down Vitest config, - you should expose it as a `config` property.
8794

8895
::: warning
89-
Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner`` and `BenchmarkRunner`).
96+
Vitest also injects an instance of `ViteNodeRunner` as `__vitest_executor` property. You can use it to process files in `importFile` method (this is default behavior of `TestRunner` and `BenchmarkRunner`).
9097

9198
`ViteNodeRunner` exposes `executeId` method, which is used to import test files in a Vite-friendly environment. Meaning, it will resolve imports and transform file content at runtime so that Node can understand it.
9299
:::

‎docs/config/index.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1375,3 +1375,13 @@ Influences whether or not the `showDiff` flag should be included in the thrown A
13751375
Sets length threshold for actual and expected values in assertion errors. If this threshold is exceeded, for example for large data structures, the value is replaced with something like `[ Array(3) ]` or `{ Object (prop1, prop2) }`. Set it to `0` if you want to disable truncating altogether.
13761376

13771377
This config option affects truncating values in `test.each` titles and inside the assertion error message.
1378+
1379+
### bail
1380+
1381+
- **Type:** `number`
1382+
- **Default:** `0`
1383+
- **CLI**: `--bail=<value>`
1384+
1385+
Stop test execution when given number of tests have failed.
1386+
1387+
By default Vitest will run all of your test cases even if some of them fail. This may not be desired for CI builds where you are only interested in 100% successful builds and would like to stop test execution as early as possible when test failures occur. The `bail` option can be used to speed up CI runs by preventing it from running more tests when failures have occured.

‎docs/guide/cli.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ Run only [benchmark](https://vitest.dev/guide/features.html#benchmarking-experim
9292
| `--no-color` | Removes colors from the console output |
9393
| `--inspect` | Enables Node.js inspector |
9494
| `--inspect-brk` | Enables Node.js inspector with break |
95+
| `--bail <number>` | Stop test execution when given number of tests have failed |
9596
| `-h, --help` | Display available CLI options |
9697

9798
::: tip

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
"test:run": "vitest run -r test/core",
2424
"test:all": "CI=true pnpm -r --stream run test --allowOnly",
2525
"test:ci": "CI=true pnpm -r --stream --filter !test-fails --filter !test-browser --filter !test-esm --filter !test-browser run test --allowOnly",
26-
"test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
26+
"test:ci:single-thread": "CI=true pnpm -r --stream --filter !test-fails --filter !test-coverage --filter !test-watch --filter !test-bail --filter !test-esm --filter !test-browser run test --allowOnly --no-threads",
2727
"typecheck": "tsc --noEmit",
2828
"typecheck:why": "tsc --noEmit --explainFiles > explainTypes.txt",
2929
"ui:build": "vite build packages/ui",

‎packages/browser/src/client/main.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createClient } from '@vitest/ws-client'
22
// eslint-disable-next-line no-restricted-imports
33
import type { ResolvedConfig } from 'vitest'
4-
import type { VitestRunner } from '@vitest/runner'
4+
import type { CancelReason, VitestRunner } from '@vitest/runner'
55
import { createBrowserRunner } from './runner'
66
import { importId } from './utils'
77
import { setupConsoleLogSpy } from './logger'
@@ -30,7 +30,16 @@ function getQueryPaths() {
3030
return url.searchParams.getAll('path')
3131
}
3232

33-
export const client = createClient(ENTRY_URL)
33+
let setCancel = (_: CancelReason) => {}
34+
const onCancel = new Promise<CancelReason>((resolve) => {
35+
setCancel = resolve
36+
})
37+
38+
export const client = createClient(ENTRY_URL, {
39+
handlers: {
40+
onCancel: setCancel,
41+
},
42+
})
3443

3544
const ws = client.ws
3645

@@ -103,6 +112,10 @@ async function runTests(paths: string[], config: ResolvedConfig) {
103112
runner = new BrowserRunner({ config, browserHashMap })
104113
}
105114

115+
onCancel.then((reason) => {
116+
runner?.onCancel?.(reason)
117+
})
118+
106119
if (!config.snapshotOptions.snapshotEnvironment)
107120
config.snapshotOptions.snapshotEnvironment = new BrowserSnapshotEnvironment()
108121

‎packages/browser/src/client/runner.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,20 @@ export function createBrowserRunner(original: any, coverageModule: CoverageHandl
2323
}
2424

2525
async onAfterRunTest(task: Test) {
26-
await super.onAfterRunTest?.()
26+
await super.onAfterRunTest?.(task)
2727
task.result?.errors?.forEach((error) => {
2828
console.error(error.message)
2929
})
30+
31+
if (this.config.bail && task.result?.state === 'fail') {
32+
const previousFailures = await rpc().getCountOfFailedTests()
33+
const currentFailures = 1 + previousFailures
34+
35+
if (currentFailures >= this.config.bail) {
36+
rpc().onCancel('test-failure')
37+
this.onCancel?.('test-failure')
38+
}
39+
}
3040
}
3141

3242
async onAfterRunSuite() {

‎packages/runner/src/types/runner.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ export interface VitestRunnerConstructor {
2727
new(config: VitestRunnerConfig): VitestRunner
2828
}
2929

30+
export type CancelReason = 'keyboard-input' | 'test-failure' | string & {}
31+
3032
export interface VitestRunner {
3133
/**
3234
* First thing that's getting called before actually collecting and running tests.
@@ -37,6 +39,13 @@ export interface VitestRunner {
3739
*/
3840
onCollected?(files: File[]): unknown
3941

42+
/**
43+
* Called when test runner should cancel next test runs.
44+
* Runner should listen for this method and mark tests and suites as skipped in
45+
* "onBeforeRunSuite" and "onBeforeRunTest" when called.
46+
*/
47+
onCancel?(reason: CancelReason): unknown
48+
4049
/**
4150
* Called before running a single test. Doesn't have "result" yet.
4251
*/

‎packages/vitest/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@
155155
"std-env": "^3.3.2",
156156
"strip-literal": "^1.0.1",
157157
"tinybench": "^2.4.0",
158-
"tinypool": "^0.4.0",
158+
"tinypool": "^0.5.0",
159159
"vite": "^3.0.0 || ^4.0.0",
160160
"vite-node": "workspace:*",
161161
"why-is-node-running": "^2.2.2"

‎packages/vitest/src/api/setup.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,16 +111,24 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, server?: Vit
111111
return ctx.updateSnapshot()
112112
return ctx.updateSnapshot([file.filepath])
113113
},
114+
onCancel(reason) {
115+
ctx.cancelCurrentRun(reason)
116+
},
117+
getCountOfFailedTests() {
118+
return ctx.state.getCountOfFailedTests()
119+
},
114120
},
115121
{
116122
post: msg => ws.send(msg),
117123
on: fn => ws.on('message', fn),
118-
eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected'],
124+
eventNames: ['onUserConsoleLog', 'onFinished', 'onCollected', 'onCancel'],
119125
serialize: stringify,
120126
deserialize: parse,
121127
},
122128
)
123129

130+
ctx.onCancel(reason => rpc.onCancel(reason))
131+
124132
clients.set(ws, rpc)
125133

126134
ws.on('close', () => {

‎packages/vitest/src/api/types.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { TransformResult } from 'vite'
2+
import type { CancelReason } from '@vitest/runner'
23
import type { AfterSuiteRunMeta, File, ModuleGraphData, Reporter, ResolvedConfig, SnapshotResult, TaskResultPack, UserConsoleLog } from '../types'
34

45
export interface TransformResultWithSource extends TransformResult {
@@ -10,6 +11,8 @@ export interface WebSocketHandlers {
1011
onTaskUpdate(packs: TaskResultPack[]): void
1112
onAfterSuiteRun(meta: AfterSuiteRunMeta): void
1213
onDone(name: string): void
14+
onCancel(reason: CancelReason): void
15+
getCountOfFailedTests(): number
1316
sendLog(log: UserConsoleLog): void
1417
getFiles(): File[]
1518
getPaths(): string[]
@@ -28,4 +31,5 @@ export interface WebSocketHandlers {
2831
}
2932

3033
export interface WebSocketEvents extends Pick<Reporter, 'onCollected' | 'onFinished' | 'onTaskUpdate' | 'onUserConsoleLog' | 'onPathsCollected'> {
34+
onCancel(reason: CancelReason): void
3135
}

0 commit comments

Comments
 (0)