Remove MDX processor on buildEnd (#10770)

bluwy · web-flow · commit 88ee63a3ba44 · 2024-04-15T15:56:53.000+08:00
diff --git a/.changeset/orange-ladybugs-nail.md b/.changeset/orange-ladybugs-nail.md
@@ -0,0 +1,5 @@
+---
+"@astrojs/mdx": patch
+---
+
+Removes internal MDX processor on `buildEnd` to free up memory
diff --git a/benchmark/make-project/memory-default.js b/benchmark/make-project/memory-default.js
@@ -35,6 +35,17 @@ ${loremIpsum}
 		);
 	}
 
+	for (let i = 0; i < 100; i++) {
+		const content = `\
+# Post ${i}
+
+${loremIpsum}
+`;
+		promises.push(
+			fs.writeFile(new URL(`./src/content/blog/post-${i}.mdx`, projectDir), content, 'utf-8')
+		);
+	}
+
 	await fs.writeFile(
 		new URL(`./src/pages/blog/[...slug].astro`, projectDir),
 		`\
@@ -56,4 +67,16 @@ const { Content } = await entry.render();
 	);
 
 	await Promise.all(promises);
+
+	await fs.writeFile(
+		new URL('./astro.config.js', projectDir),
+		`\
+import { defineConfig } from 'astro/config';
+import mdx from '@astrojs/mdx';
+
+export default defineConfig({
+  integrations: [mdx()],
+});`,
+		'utf-8'
+	);
 }
diff --git a/packages/astro/src/vite-plugin-markdown/index.ts b/packages/astro/src/vite-plugin-markdown/index.ts
@@ -55,7 +55,15 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
 
 				const fileURL = pathToFileURL(fileId);
 
-				const renderResult = await processor!
+				// `processor` is initialized in `buildStart`, and removed in `buildEnd`. `load`
+				// should be called in between those two lifecycles, so this error should never happen
+				if (!processor) {
+					return this.error(
+						'MDX processor is not initialized. This is an internal error. Please file an issue.'
+					);
+				}
+
+				const renderResult = await processor
 					.render(raw.content, {
 						// @ts-expect-error passing internal prop
 						fileURL,
diff --git a/packages/integrations/mdx/src/index.ts b/packages/integrations/mdx/src/index.ts
@@ -75,14 +75,17 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 					),
 				});
 
-				let processor: ReturnType<typeof createMdxProcessor>;
+				let processor: ReturnType<typeof createMdxProcessor> | undefined;
 
 				updateConfig({
 					vite: {
 						plugins: [
 							{
 								name: '@mdx-js/rollup',
 								enforce: 'pre',
+								buildEnd() {
+									processor = undefined;
+								},
 								configResolved(resolved) {
 									processor = createMdxProcessor(mdxOptions, {
 										sourcemap: !!resolved.build.sourcemap,
@@ -118,6 +121,14 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 									// Ensure `data.astro` is available to all remark plugins
 									setVfileFrontmatter(vfile, frontmatter);
 
+									// `processor` is initialized in `configResolved`, and removed in `buildEnd`. `transform`
+									// should be called in between those two lifecycle, so this error should never happen
+									if (!processor) {
+										return this.error(
+											'MDX processor is not initialized. This is an internal error. Please file an issue.'
+										);
+									}
+
 									try {
 										const compiled = await processor.process(vfile);
 
diff --git a/packages/markdown/remark/src/highlight.ts b/packages/markdown/remark/src/highlight.ts
@@ -73,6 +73,7 @@ export async function highlightCodeBlocks(tree: Root, highlighter: Highlighter)
 	for (const { node, language, grandParent, parent } of nodes) {
 		const meta = (node.data as any)?.meta ?? node.properties.metastring ?? undefined;
 		const code = toText(node, { whitespace: 'pre' });
+		// TODO: In Astro 5, have `highlighter()` return hast directly to skip expensive HTML parsing and serialization.
 		const html = await highlighter(code, language, { meta });
 		// The replacement returns a root node with 1 child, the `<pr>` element replacement.
 		const replacement = fromHtml(html, { fragment: true }).children[0] as Element;

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"@astrojs/mdx": patch
 +---
++
 +Removes internal MDX processor on `buildEnd` to free up memory