Skip to content

Commit 8d88738

Browse files
Jaciezytfarnabaz
Jaciezyt
and
farnabaz
authored
feat: single CSV file collections (#3513)
Co-authored-by: Farnabaz <farnabaz@gmail.com>
1 parent 1ef7768 commit 8d88738

File tree

17 files changed

+369
-31
lines changed

17 files changed

+369
-31
lines changed

‎docs/content/docs/3.files/4.csv.md‎

Lines changed: 89 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,76 +3,136 @@ title: CSV
33
description: How to define, write and query CSV data.
44
---
55

6-
## Define Collection
6+
## Single-file source
7+
8+
When you point a collection to a single CSV file (instead of a glob), Nuxt Content **treats each data row as a separate item** in the collection.
9+
10+
- **Define the collection**: set `source` to the path of a single `.csv` file.
11+
- **Item generation**: each data row becomes an item with the row’s fields at the top level (no `body` array).
12+
- **IDs**: item IDs are suffixed with `#<rowNumber>`, where `#1` is the first data row after the header.
713

814
```ts [content.config.ts]
915
import { defineCollection, defineContentConfig } from '@nuxt/content'
1016
import { z } from 'zod'
1117

1218
export default defineContentConfig({
1319
collections: {
14-
authors: defineCollection({
20+
people: defineCollection({
1521
type: 'data',
16-
source: 'authors/**.csv',
22+
source: 'org/people.csv',
1723
schema: z.object({
1824
name: z.string(),
19-
email: z.string(),
20-
avatar: z.string()
25+
email: z.string().email()
2126
})
2227
})
2328
}
2429
})
30+
```
31+
32+
```csv [content/org/people.csv]
33+
name,email
34+
Alice,alice@example.com
35+
Bob,bob@example.com
36+
```
37+
38+
Each row produces its own item. For example, the first data row will have an ID ending with `#1` and the second with `#2`. You can query by any column:
39+
40+
```ts
41+
const { data: alice } = await useAsyncData('alice', () =>
42+
queryCollection('people')
43+
.where('email', '=', 'alice@example.com')
44+
.first()
45+
)
2546

47+
const { data: allPeople } = await useAsyncData('all-people', () =>
48+
queryCollection('people')
49+
.order('name', 'ASC')
50+
.all()
51+
)
2652
```
2753

28-
## Create `.csv` files
54+
::note
55+
- The header row is required and is not turned into an item.
56+
- With a single-file source, items contain row fields at the top level (no `body`).
57+
- If you prefer treating each CSV file as a single item containing all rows in `body`, use a glob source like `org/**.csv` instead of a single file.
58+
:::
59+
60+
## Multiple-files source
2961

30-
Create author files in `content/authors/` directory.
62+
If you uses `*/**.csv` as source in configuration, Nuxt Content will treat them differently from single-file collections.
63+
**Each file(not row) will be treated as an item**, rows will be parsed into `body` field in item object as an array.
64+
65+
```ts [content.config.ts]
66+
import { defineCollection, defineContentConfig } from '@nuxt/content'
67+
import { z } from 'zod'
68+
69+
export default defineContentConfig({
70+
collections: {
71+
charts: defineCollection({
72+
type: 'data',
73+
source: 'charts/**.csv',
74+
schema: z.object({
75+
// Body is important in CSV files, without body field you cannot access to data array
76+
body: z.array(z.object({
77+
label: z.string(),
78+
value: z.number()
79+
}))
80+
})
81+
})
82+
}
83+
})
84+
85+
```
86+
87+
Create chart files in `content/charts/` directory.
3188

3289
::code-group
33-
```csv [users.csv]
34-
id,name,email
35-
1,John Doe,john@example.com
36-
2,Jane Smith,jane@example.com
37-
3,Alice Johnson,alice@example.com
90+
```csv [content/charts/chart1.csv]
91+
label,value
92+
A,100
93+
B,200
94+
C,300
3895
```
3996

40-
```csv [team.csv]
41-
name,role,avatar
42-
John Doe,Developer,https://avatars.githubusercontent.com/u/1?v=4
43-
Jane Smith,Designer,https://avatars.githubusercontent.com/u/2?v=4
97+
```csv [content/charts/chart2.csv]
98+
label,value
99+
Foo,123
100+
Bar,456
101+
Baz,789
44102
```
45103
::
46104

47105
::warning
48106
Each CSV file should have a header row that defines the column names, which will be used as object keys when parsed.
49107
::
50108

51-
## Query Data
52-
53-
Now we can query authors:
109+
Now we can query charts:
54110

55111
```vue
56112
<script lang="ts" setup>
57-
// Find a single author
58-
const { data: author } = await useAsyncData('john-doe', () => {
59-
return queryCollection('authors')
60-
.where('name', '=', 'John Doe')
113+
// Find a single chart
114+
const { data: chart1 } = await useAsyncData('chart1', () => {
115+
return queryCollection('charts')
116+
.where('id', '=', 'charts/charts/chart1.csv')
61117
.first()
62118
})
63119
64-
// Get all authors
65-
const { data: authors } = await useAsyncData('authors', () => {
66-
return queryCollection('authors')
67-
.order('name', 'ASC')
120+
// Get all charts
121+
const { data: charts } = await useAsyncData('charts', () => {
122+
return queryCollection('charts')
123+
.order('id', 'ASC')
68124
.all()
69125
})
126+
70127
</script>
71128
72129
<template>
73130
<ul>
74-
<li v-for="author in authors" :key="author.id">
75-
{{ author.name }} ({{ author.email }})
131+
<li v-for="chart in charts" :key="chart.id">
132+
<!-- CSV data are in `chart.body` as an array -->
133+
<p v-for="data in chart.body">
134+
{{ data.label }} - {{ data.value }}
135+
</p>
76136
</li>
77137
</ul>
78138
</template>

‎playground/content.config.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,21 @@ const pages = defineCollection({
6767
})
6868

6969
const collections = {
70+
people: defineCollection({
71+
type: 'data',
72+
source: 'org/people.csv',
73+
schema: z.object({
74+
name: z.string(),
75+
email: z.string().email(),
76+
}),
77+
}),
78+
org: defineCollection({
79+
type: 'data',
80+
source: 'org/**.csv',
81+
schema: z.object({
82+
body: z.array(z.any()),
83+
}),
84+
}),
7085
hackernews,
7186
content,
7287
data,

‎playground/content/org/people.csv‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
name,email
2+
John Doe,john.doe@example.com
3+
Jane Smith,jane.smith@example.com
4+
Bob Johnson,bob.johnson@example.com
5+
Alice Brown,alice.brown@example.com
6+
Charlie Wilson,charlie.wilson@example.com
7+
Diana Lee,diana.lee@example.com
8+
Eve Davis,eve.davis@example.com
9+
Frank Miller,frank.miller@example.com
10+
Grace Taylor,grace.taylor@example.com
11+
Henry Anderson,henry.anderson@example.com

‎playground/pages/org/data.vue‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<script setup lang="ts">
2+
const { data } = await useAsyncData('tmp-content', () => queryCollection('org').all())
3+
</script>
4+
5+
<template>
6+
<div>
7+
<h1>People</h1>
8+
<pre>{{ data }}</pre>
9+
</div>
10+
</template>

‎playground/pages/org/people.vue‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<script setup lang="ts">
2+
const { data: tmpContent } = await useAsyncData('tmp-content', () => queryCollection('people').all())
3+
</script>
4+
5+
<template>
6+
<div>
7+
<h1>People</h1>
8+
<pre>{{ tmpContent }}</pre>
9+
</div>
10+
</template>

‎pnpm-lock.yaml‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/utils/content/index.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ async function _getHighlightPlugin(key: string, options: HighlighterOptions) {
112112
export async function createParser(collection: ResolvedCollection, nuxt?: Nuxt) {
113113
const nuxtOptions = nuxt?.options as unknown as { content: ModuleOptions, mdc: MDCModuleOptions }
114114
const mdcOptions = nuxtOptions?.mdc || {}
115-
const { pathMeta = {}, markdown = {}, transformers = [] } = nuxtOptions?.content?.build || {}
115+
const { pathMeta = {}, markdown = {}, transformers = [], csv = {}, yaml = {} } = nuxtOptions?.content?.build || {}
116116

117117
const rehypeHighlightPlugin = markdown.highlight !== false
118118
? await getHighlightPluginInstance(defu(markdown.highlight as HighlighterOptions, mdcOptions.highlight, { compress: true }))
@@ -150,6 +150,8 @@ export async function createParser(collection: ResolvedCollection, nuxt?: Nuxt)
150150
},
151151
highlight: undefined,
152152
},
153+
csv: csv,
154+
yaml: yaml,
153155
}
154156

155157
return async function parse(file: ContentFile) {

‎src/utils/content/transformers/csv/index.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@ export default defineTransformer({
5353
})
5454
const { result } = await stream.process(file.body)
5555

56+
// If the file id includes a row index, parse the file as a single object
57+
if (file.id.includes('#')) {
58+
return { id: file.id, ...result[0] }
59+
}
60+
61+
// Otherwise, parse the file as an array
5662
return {
5763
id: file.id,
5864
body: result,

‎src/utils/schema/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export function detectSchemaVendor(schema: ContentStandardSchemaV1) {
106106
}
107107

108108
export function replaceComponentSchemas<T = Draft07Definition | Draft07DefinitionProperty>(property: T): T {
109-
if ((property as Draft07DefinitionProperty).type === 'array') {
109+
if ((property as Draft07DefinitionProperty).type === 'array' && (property as Draft07DefinitionProperty).items) {
110110
(property as Draft07DefinitionProperty).items = replaceComponentSchemas((property as Draft07DefinitionProperty).items as Draft07DefinitionProperty) as Draft07DefinitionProperty
111111
}
112112

‎src/utils/source.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { readFile } from 'node:fs/promises'
2+
import { createReadStream } from 'node:fs'
23
import { join, normalize } from 'pathe'
34
import { withLeadingSlash, withoutTrailingSlash } from 'ufo'
45
import { glob } from 'tinyglobby'
@@ -20,6 +21,12 @@ export function defineLocalSource(source: CollectionSource | ResolvedCollectionS
2021
logger.warn('Collection source should not start with `./` or `../`.')
2122
source.include = source.include.replace(/^(\.\/|\.\.\/|\/)*/, '')
2223
}
24+
25+
// If source is a CSV file, define a CSV source
26+
if (source.include.endsWith('.csv') && !source.include.includes('*')) {
27+
return defineCSVSource(source)
28+
}
29+
2330
const { fixed } = parseSourceBase(source)
2431
const resolvedSource: ResolvedCollectionSource = {
2532
_resolved: true,
@@ -97,6 +104,69 @@ export function defineGitSource(source: CollectionSource): ResolvedCollectionSou
97104
return resolvedSource
98105
}
99106

107+
export function defineCSVSource(source: CollectionSource): ResolvedCollectionSource {
108+
const { fixed } = parseSourceBase(source)
109+
110+
const resolvedSource: ResolvedCollectionSource = {
111+
_resolved: true,
112+
prefix: withoutTrailingSlash(withLeadingSlash(fixed)),
113+
prepare: async ({ rootDir }) => {
114+
resolvedSource.cwd = source.cwd
115+
? String(normalize(source.cwd)).replace(/^~~\//, rootDir)
116+
: join(rootDir, 'content')
117+
},
118+
getKeys: async () => {
119+
const _keys = await glob(source.include, { cwd: resolvedSource.cwd, ignore: getExcludedSourcePaths(source), dot: true, expandDirectories: false })
120+
.catch((): [] => [])
121+
const keys = _keys.map(key => key.substring(fixed.length))
122+
if (keys.length !== 1) {
123+
return keys
124+
}
125+
126+
return new Promise((resolve) => {
127+
const csvKeys: string[] = []
128+
let count = 0
129+
let lastByteWasNewline = true
130+
createReadStream(join(resolvedSource.cwd, fixed, keys[0]!))
131+
.on('data', function (chunk) {
132+
for (let i = 0; i < chunk.length; i += 1) {
133+
if (chunk[i] == 10) {
134+
if (count > 0) { // count === 0 is CSV header row and should not be included
135+
csvKeys.push(`${keys[0]}#${count}`)
136+
}
137+
count += 1
138+
}
139+
lastByteWasNewline = chunk[i] == 10
140+
}
141+
})
142+
.on('end', () => {
143+
// If file doesn't end with newline and we have at least one data row, add the last row
144+
if (!lastByteWasNewline && count > 0) {
145+
csvKeys.push(`${keys[0]}#${count}`)
146+
}
147+
resolve(csvKeys)
148+
})
149+
})
150+
},
151+
getItem: async (key) => {
152+
const [csvKey, csvIndex] = key.split('#')
153+
const fullPath = join(resolvedSource.cwd, fixed, csvKey!)
154+
const content = await readFile(fullPath, 'utf8')
155+
156+
if (key.includes('#')) {
157+
const lines = content.split('\n')
158+
return lines[0] + '\n' + lines[+(csvIndex || 0)]!
159+
}
160+
161+
return content
162+
},
163+
...source,
164+
include: source.include,
165+
cwd: '',
166+
}
167+
return resolvedSource
168+
}
169+
100170
export function parseSourceBase(source: CollectionSource) {
101171
const [fixPart, ...rest] = source.include.includes('*') ? source.include.split('*') : ['', source.include]
102172
return {

0 commit comments

Comments
 (0)