fix(runner): make the default value of retry and repeats 0 (#3638)

Dunqing · web-flow · commit 6d146d16900d · 2023-06-22T16:02:08.000+02:00
diff --git a/docs/api/index.md b/docs/api/index.md
@@ -17,12 +17,16 @@ interface TestOptions {
   timeout?: number
   /**
    * Will retry the test specific number of times if it fails
+   *
+   * @default 0
    */
   retry?: number
   /**
    * Will repeat the same test several times even if it fails each time
    * If you have "retry" option and it fails, it will use every retry in each cycle
    * Useful for debugging random failings
+   *
+   * @default 0
    */
   repeats?: number
 }
diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts
@@ -127,22 +127,20 @@ export async function runTest(test: Test, runner: VitestRunner) {
   test.result = {
     state: 'run',
     startTime: start,
+    retryCount: 0,
   }
   updateTask(test, runner)
 
   setCurrentTest(test)
 
-  const repeats = typeof test.repeats === 'number' ? test.repeats : 1
-
-  for (let repeatCount = 0; repeatCount < repeats; repeatCount++) {
-    const retry = test.retry || 1
-
-    for (let retryCount = 0; retryCount < retry; retryCount++) {
+  const repeats = test.repeats ?? 0
+  for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) {
+    const retry = test.retry ?? 0
+    for (let retryCount = 0; retryCount <= retry; retryCount++) {
       let beforeEachCleanups: HookCleanupCallback[] = []
       try {
         await runner.onBeforeTryTest?.(test, { retry: retryCount, repeats: repeatCount })
 
-        test.result.retryCount = retryCount
         test.result.repeatCount = repeatCount
 
         beforeEachCleanups = await callSuiteHook(test.suite, test, 'beforeEach', runner, [test.context, test.suite])
@@ -188,9 +186,10 @@ export async function runTest(test: Test, runner: VitestRunner) {
       if (test.result.state === 'pass')
         break
 
-      if (retryCount < retry - 1) {
+      if (retryCount < retry) {
         // reset state when retry test
         test.result.state = 'run'
+        test.result.retryCount = (test.result.retryCount ?? 0) + 1
       }
 
       // update retry info
diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts
@@ -166,15 +166,14 @@ export interface TestOptions {
    * Times to retry the test if fails. Useful for making flaky tests more stable.
    * When retries is up, the last test error will be thrown.
    *
-   * @default 1
+   * @default 0
    */
   retry?: number
   /**
    * How many times the test will run.
    * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default.
    *
-   * @default 1
-   *
+   * @default 0
    */
   repeats?: number
 }
diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts
@@ -100,7 +100,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (level === 0 && task.type === 'suite' && task.projectName)
       prefix += formatProjectName(task.projectName)
 
-    if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 1)
+    if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 0)
       suffix += c.yellow(` (retry x${task.result.retryCount})`)
 
     if (task.type === 'suite' && !task.meta?.typecheck) {
@@ -111,7 +111,7 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (task.mode === 'skip' || task.mode === 'todo')
       suffix += ` ${c.dim(c.gray('[skipped]'))}`
 
-    if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 1)
+    if (task.type === 'test' && task.result?.repeatCount && task.result.repeatCount > 0)
       suffix += c.yellow(` (repeat x${task.result.repeatCount})`)
 
     if (task.result?.duration != null) {
diff --git a/test/core/test/propagate-options-nested-suite.test.ts b/test/core/test/propagate-options-nested-suite.test.ts
@@ -19,5 +19,5 @@ describe(
       })
     })
   },
-  { retry: 10 },
+  { retry: 9 },
 )
diff --git a/test/core/test/repeats.test.ts b/test/core/test/repeats.test.ts
@@ -1,3 +1,4 @@
+import { getCurrentTest } from '@vitest/runner'
 import { afterAll, describe, expect, test } from 'vitest'
 
 const testNumbers: number[] = []
@@ -7,16 +8,16 @@ describe('testing it/test', () => {
 
   test('test 1', () => {
     testNumbers.push(1)
-  }, { repeats: 5 })
+  }, { repeats: 4 })
 
   test('test 2', () => {
     testNumbers.push(2)
-  }, { repeats: 3 })
+  }, { repeats: 2 })
 
   test.fails('test 3', () => {
     testNumbers.push(3)
     expect(testNumbers).toStrictEqual(result)
-  }, { repeats: 1 })
+  }, { repeats: 0 })
 
   afterAll(() => {
     result.push(3)
@@ -30,7 +31,7 @@ describe('testing describe', () => {
   test('test 1', () => {
     describeNumbers.push(1)
   })
-}, { repeats: 3 })
+}, { repeats: 2 })
 
 afterAll(() => {
   expect(describeNumbers).toStrictEqual([1, 1, 1])
@@ -39,14 +40,21 @@ afterAll(() => {
 const retryNumbers: number[] = []
 
 describe('testing repeats with retry', () => {
-  const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
-  test('test 1', () => {
-    retryNumbers.push(1)
-  }, { repeats: 5, retry: 2 })
-
-  afterAll(() => {
-    expect(retryNumbers).toStrictEqual(result)
+  describe('normal test', () => {
+    const result = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+    test.fails('test 1', () => {
+      retryNumbers.push(1)
+      expect(1).toBe(2)
+    }, { repeats: 4, retry: 1 })
+
+    afterAll(() => {
+      expect(retryNumbers).toStrictEqual(result)
+    })
   })
+
+  test('should not reset retry count', () => {
+    expect(getCurrentTest()!.result?.retryCount).toBe(3)
+  }, { repeats: 2, retry: 1 })
 })
 
 const nestedDescribeNumbers: number[] = []
@@ -71,10 +79,10 @@ describe('testing nested describe', () => {
           nestedDescribeNumbers.push(4)
         })
       }, 100)
-    }, { repeats: 3 })
+    }, { repeats: 2 })
   })
 
   afterAll(() => {
     expect(nestedDescribeNumbers).toStrictEqual([1, 1, 2, 2, 3, 3, 3, 4, 4, 4])
   })
-}, { repeats: 2 })
+}, { repeats: 1 })
diff --git a/test/core/test/retry-only.test.ts b/test/core/test/retry-only.test.ts
@@ -11,5 +11,5 @@ describe.only('description.only retry', () => {
   it('test should not inherit options from the description block if exists', () => {
     count5 += 1
     expect(count5).toBe(5)
-  }, { retry: 5 })
-}, { retry: 2 })
+  }, { retry: 4 })
+}, { retry: 1 })
diff --git a/test/core/test/retry.test.ts b/test/core/test/retry.test.ts
@@ -4,13 +4,13 @@ let count1 = 0
 it('retry test', () => {
   count1 += 1
   expect(count1).toBe(3)
-}, { retry: 3 })
+}, { retry: 2 })
 
 let count2 = 0
 it.fails('retry test fails', () => {
   count2 += 1
   expect(count2).toBe(3)
-}, { retry: 2 })
+}, { retry: 1 })
 
 let count3 = 0
 it('retry test fails', () => {
diff --git a/test/failing/fixtures/expects/soft.test.ts b/test/failing/fixtures/expects/soft.test.ts
@@ -73,13 +73,13 @@ test('retry will passed', () => {
   expect.soft(num += 1).toBe(3)
   expect.soft(num += 1).toBe(4)
 }, {
-  retry: 2,
+  retry: 1,
 })
 
 num = 0
 test('retry will failed', () => {
   expect.soft(num += 1).toBe(4)
   expect.soft(num += 1).toBe(5)
 }, {
-  retry: 2,
+  retry: 1,
 })

Original file line number	Diff line number	Diff line change
`@@ -166,15 +166,14 @@ export interface TestOptions {`
`166`	`166`	`* Times to retry the test if fails. Useful for making flaky tests more stable.`
`167`	`167`	`* When retries is up, the last test error will be thrown.`
`168`	`168`	`*`
`169`		`- * @default 1`
	`169`	`+ * @default 0`
`170`	`170`	`*/`
`171`	`171`	`retry?: number`
`172`	`172`	`/**`
`173`	`173`	`* How many times the test will run.`
`174`	`174`	* Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default.
`175`	`175`	`*`
`176`		`- * @default 1`
`177`		`- *`
	`176`	`+ * @default 0`
`178`	`177`	`*/`
`179`	`178`	`repeats?: number`
`180`	`179`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,5 +19,5 @@ describe(`
`19`	`19`	`})`
`20`	`20`	`})`
`21`	`21`	`},`
`22`		`- { retry: 10 },`
	`22`	`+ { retry: 9 },`
`23`	`23`	`)`