Skip to content

Commit e60b2f4

Browse files
hi-ogawaclaudecodexAriPerkkio
authored
feat: support merge reports for non-sharded multi-environment runs (take 2) (#10031)
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Ari Perkkiö <ari.perkkio@gmail.com>
1 parent c99d18f commit e60b2f4

39 files changed

Lines changed: 902 additions & 203 deletions

‎.github/workflows/ci.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ jobs:
116116

117117
- name: Test
118118
run: pnpm run test:ci
119+
env:
120+
VITEST_CI_BLOB_LABEL: ${{ matrix.os }}-node-${{ matrix.node_version }}
119121

120122
- name: Test Examples
121123
run: pnpm run test:examples
@@ -130,6 +132,17 @@ jobs:
130132
path: test/ui/test-results/
131133
retention-days: 30
132134

135+
- uses: actions/upload-artifact@v7
136+
if: ${{ !cancelled() }}
137+
with:
138+
name: vitest-blob-${{ matrix.os }}-node-${{ matrix.node_version }}
139+
path: |
140+
README.md
141+
test/unit/.vitest-reports
142+
test/e2e/.vitest-reports
143+
retention-days: 1
144+
include-hidden-files: true
145+
133146
test-cached:
134147
needs: changed
135148
name: 'Cache&Test: node-${{ matrix.node_version }}, ${{ matrix.os }}'
@@ -251,3 +264,49 @@ jobs:
251264
name: playwright-report-rolldown
252265
path: rolldown/test/ui/test-results/
253266
retention-days: 30
267+
268+
merge-reports:
269+
needs: test
270+
if: ${{ !cancelled() }}
271+
runs-on: ubuntu-latest
272+
name: Merge Reports
273+
timeout-minutes: 10
274+
steps:
275+
- uses: actions/checkout@v6
276+
277+
- uses: ./.github/actions/setup-and-cache
278+
279+
- name: Install
280+
run: pnpm i
281+
282+
- name: Build
283+
run: pnpm run build
284+
285+
- uses: actions/download-artifact@v4
286+
with:
287+
pattern: vitest-blob-*
288+
merge-multiple: true
289+
290+
- name: Merge reports
291+
continue-on-error: true
292+
run: pnpm --filter=./test/unit --filter=./test/e2e --no-bail --sequential test --merge-reports --reporter=html
293+
294+
- name: Merge reports html
295+
id: merge-html
296+
run: |
297+
mkdir -p html-all
298+
cp -rf test/unit/html html-all/unit
299+
cp -rf test/e2e/html html-all/e2e
300+
echo "short_sha=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
301+
302+
- uses: actions/upload-artifact@v7
303+
id: upload-report
304+
with:
305+
name: vitest-ci-report-${{ steps.merge-html.outputs.short_sha }}
306+
path: html-all
307+
retention-days: 7
308+
309+
- name: Link report viewer
310+
run: |
311+
echo "::notice title=Vitest HTML report::View HTML report: https://viewer.vitest.dev/?url=${{ steps.upload-report.outputs.artifact-url }}"
312+
echo "[View HTML report](https://viewer.vitest.dev/?url=${{ steps.upload-report.outputs.artifact-url }})" >> $GITHUB_STEP_SUMMARY

‎docs/guide/cli.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ You cannot use this option with `--watch` enabled (enabled in dev by default).
225225
:::
226226

227227
::: tip
228-
If `--reporter=blob` is used without an output file, the default path will include the current shard config to avoid collisions with other Vitest processes.
228+
If `--reporter=blob` is used without an output file, the default path will include the current shard config and blob label from `VITEST_BLOB_LABEL` or the blob reporter `label` option to avoid collisions with other Vitest processes.
229229
:::
230230

231231
### merge-reports

‎docs/guide/improving-performance.md‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,12 @@ Collect the results stored in `.vitest-reports` directory from each machine and
132132
vitest run --merge-reports
133133
```
134134

135+
When running the same shards across multiple environments, set the `VITEST_BLOB_LABEL` environment variable so merged reports can display them separately:
136+
137+
```sh
138+
VITEST_BLOB_LABEL=linux vitest run --reporter=blob --shard=1/3
139+
```
140+
135141
::: details GitHub Actions example
136142
This setup is also used at https://github.com/vitest-tests/test-sharding.
137143

@@ -144,9 +150,10 @@ on:
144150
- main
145151
jobs:
146152
tests:
147-
runs-on: ubuntu-latest
153+
runs-on: ${{ matrix.os }}
148154
strategy:
149155
matrix:
156+
os: [ubuntu-latest, macos-latest]
150157
shardIndex: [1, 2, 3, 4]
151158
shardTotal: [4]
152159
steps:
@@ -163,12 +170,14 @@ jobs:
163170

164171
- name: Run tests
165172
run: pnpm run test --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
173+
env:
174+
VITEST_BLOB_LABEL: ${{ matrix.os }}
166175

167176
- name: Upload blob report to GitHub Actions Artifacts
168177
if: ${{ !cancelled() }}
169178
uses: actions/upload-artifact@v4
170179
with:
171-
name: blob-report-${{ matrix.shardIndex }}
180+
name: blob-report-${{ matrix.os }}-${{ matrix.shardIndex }}
172181
path: .vitest-reports/*
173182
include-hidden-files: true
174183
retention-days: 1
@@ -177,7 +186,7 @@ jobs:
177186
if: ${{ !cancelled() }}
178187
uses: actions/upload-artifact@v4
179188
with:
180-
name: blob-attachments-${{ matrix.shardIndex }}
189+
name: blob-attachments-${{ matrix.os }}-${{ matrix.shardIndex }}
181190
path: .vitest/**
182191
include-hidden-files: true
183192
retention-days: 1

‎docs/guide/reporters.md‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -748,13 +748,32 @@ By default, stores all results in `.vitest-reports` folder, but can be overridde
748748
npx vitest --reporter=blob --outputFile=reports/blob-1.json
749749
```
750750

751-
We recommend using this reporter if you are running Vitest on different machines with the [`--shard`](/guide/cli#shard) flag.
752-
All blob reports can be merged into any report by using `--merge-reports` command at the end of your CI pipeline:
751+
We recommend using this reporter if you are running Vitest on different machines with the [`--shard`](/guide/cli#shard) flag or across multiple environments (e.g., linux/macos/windows). All blob reports can be merged into any report by using `--merge-reports` command at the end of your CI pipeline:
753752

754753
```bash
755754
npx vitest --merge-reports=reports --reporter=json --reporter=default
756755
```
757756

757+
When running the same tests across multiple environments, use the `VITEST_BLOB_LABEL` environment variable to distinguish each environment's blob. Vitest reads labels at merge time and displays results separately:
758+
759+
```bash
760+
VITEST_BLOB_LABEL=linux vitest run --reporter=blob
761+
```
762+
763+
You can also provide the label via the blob reporter option. This has higher priority than `VITEST_BLOB_LABEL`.
764+
765+
```ts [vitest.config.ts]
766+
import { defineConfig } from 'vitest/config'
767+
768+
export default defineConfig({
769+
test: {
770+
reporters: [
771+
['blob', { label: 'linux' }],
772+
],
773+
},
774+
})
775+
```
776+
758777
Blob reporter output doesn't include file-based [attachments](/api/advanced/artifacts.html#testattachment).
759778
Make sure to merge [`attachmentsDir`](/config/attachmentsdir) separately alongside blob reports on CI when using this feature.
760779

‎packages/browser/src/client/orchestrator.ts‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,34 @@ async function getContainer(config: SerializedConfig): Promise<HTMLDivElement> {
464464
function generateFileId(file: string) {
465465
const config = getConfig()
466466
const path = relative(config.root, file)
467-
return generateHash(`${path}${config.name || ''}`)
467+
return generateFileHash(
468+
path,
469+
config.name,
470+
{
471+
typecheck: config.pool === 'typescript',
472+
__vitest_label__: config.mergeReportsLabel,
473+
},
474+
)
475+
}
476+
477+
// TODO: copied from packages/runner/src/utils/collect.ts
478+
interface HashMeta {
479+
typecheck?: boolean
480+
__vitest_label__?: string
481+
}
482+
483+
function generateFileHash(
484+
file: string,
485+
projectName: string | undefined,
486+
meta?: HashMeta,
487+
): string {
488+
const seed = [
489+
file,
490+
projectName || '',
491+
meta?.typecheck ? '__typecheck__' : '',
492+
meta?.__vitest_label__ || '',
493+
].join('\0')
494+
return generateHash(seed)
468495
}
469496

470497
function generateHash(str: string): string {

‎packages/runner/src/collect.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,14 @@ export async function collectTests(
4747

4848
const fileTags: string[] = typeof spec === 'string' ? [] : (spec.fileTags || [])
4949

50-
const file = createFileTask(filepath, config.root, config.name, runner.pool, runner.viteEnvironment)
50+
const file = createFileTask(
51+
filepath,
52+
config.root,
53+
config.name,
54+
runner.pool,
55+
runner.viteEnvironment,
56+
{ __vitest_label__: config.mergeReportsLabel },
57+
)
5158
file.tags = fileTags
5259
file.shuffle = config.sequence.shuffle
5360

‎packages/runner/src/suite.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,18 @@ function createSuiteCollector(
347347
...options,
348348
}
349349
const timeout = options.timeout ?? runner.config.testTimeout
350+
// TODO: should this be `parentTask.meta`?
351+
// currently we don't inherit
352+
// file.meta -> task.meta
353+
// file.meta -> suite.meta (see initSuite)
354+
// but we do inherit
355+
// suite.meta -> task.meta
356+
// suite.meta -> suite.meta
357+
// and also
358+
// file.tags -> task.tags
359+
// file.tags -> suite.tags
360+
// suite.tags -> suite.tags
361+
// suite.tags -> task.tags
350362
const parentMeta = currentSuite?.meta
351363
const tagMeta = tagsOptions.meta
352364
const testMeta = Object.create(null)

‎packages/runner/src/types/runner.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ export interface VitestRunnerConfig {
4646
tags: TestTagDefinition[]
4747
tagsFilter: string[] | undefined
4848
strictTags: boolean
49+
mergeReportsLabel: string | undefined
4950
}
5051

5152
/**

‎packages/runner/src/utils/collect.ts‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,23 +193,29 @@ export function calculateSuiteHash(parent: Suite): void {
193193
})
194194
}
195195

196+
interface HashMeta {
197+
typecheck?: boolean
198+
__vitest_label__?: string
199+
}
200+
196201
export function createFileTask(
197202
filepath: string,
198203
root: string,
199204
projectName: string | undefined,
200205
pool?: string,
201206
viteEnvironment?: string,
207+
meta?: HashMeta,
202208
): File {
203209
const path = relative(root, filepath)
204210
const file: File = {
205-
id: generateFileHash(path, projectName),
211+
id: generateFileHash(path, projectName, meta),
206212
name: path,
207213
fullName: path,
208214
type: 'suite',
209215
mode: 'queued',
210216
filepath,
211217
tasks: [],
212-
meta: Object.create(null),
218+
meta: Object.assign(Object.create(null), meta),
213219
projectName,
214220
file: undefined!,
215221
pool,
@@ -228,8 +234,15 @@ export function createFileTask(
228234
export function generateFileHash(
229235
file: string,
230236
projectName: string | undefined,
237+
meta?: HashMeta,
231238
): string {
232-
return /* @__PURE__ */ generateHash(`${file}${projectName || ''}`)
239+
const seed = [
240+
file,
241+
projectName || '',
242+
meta?.typecheck ? '__typecheck__' : '',
243+
meta?.__vitest_label__ || '',
244+
].join('\0')
245+
return generateHash(seed)
233246
}
234247

235248
export function findTestFileStackTrace(testFilePath: string, error: string): ParsedStack | undefined {

‎packages/ui/client/components/FileDetails.vue‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ const isTypecheck = computed(() => {
6060
return !!current.value?.meta?.typecheck
6161
})
6262
63+
const label = computed(() => current.value?.meta?.__vitest_label__)
64+
6365
function open() {
6466
const filePath = current.value?.filepath
6567
if (filePath) {
@@ -206,6 +208,7 @@ const tags = computed(() => {
206208
<div p="2" h-10 flex="~ gap-2" items-center bg-header border="b base">
207209
<StatusIcon :state="current.result?.state" :mode="current.mode" :failed-snapshot="failedSnapshot" />
208210
<div v-if="isTypecheck" v-tooltip.bottom="'This is a typecheck test. It won\'t report results of the runtime tests'" class="i-logos:typescript-icon" flex-shrink-0 />
211+
<span v-if="label" class="rounded-sm px-1 text-xs font-light bg-cyan-500/20 text-cyan-700 dark:text-cyan-300" flex-shrink-0>{{ label }}</span>
209212
<span
210213
v-if="current?.file.projectName"
211214
class="rounded-full py-0.5 px-2 text-xs font-light"

0 commit comments

Comments
 (0)