Skip to content

Commit d46ce60

Browse files
paulopacittipaulopacitti
authored
feat(markdown): highlight frontmatter and comments (#1245)
Adds Markdown frontmatter and HTML comment highlighting, including a dedicated frontmatter split path for top-of-file YAML blocks. Also updates gruvbox themes so frontmatter tokens render consistently, with tests covering the new tokenization.
1 parent f786b2a commit d46ce60

6 files changed

Lines changed: 210 additions & 4 deletions

File tree

‎lexers/markdown.go‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,83 @@
11
package lexers
22

33
import (
4+
"strings"
5+
46
. "github.com/alecthomas/chroma/v2" // nolint
57
)
68

7-
// Markdown lexer.
8-
var Markdown = Register(MustNewLexer(
9+
// Markdown lexer with YAML frontmatter and HTML comment support.
10+
var Markdown = Register(&markdownLexer{Lexer: MustNewLexer(
911
&Config{
1012
Name: "markdown",
1113
Aliases: []string{"md", "mkd"},
1214
Filenames: []string{"*.md", "*.mkd", "*.markdown"},
1315
MimeTypes: []string{"text/x-markdown"},
1416
},
1517
markdownRules,
16-
))
18+
)})
19+
20+
// markdownLexer wraps the base Markdown lexer to highlight top-of-file YAML frontmatter.
21+
type markdownLexer struct {
22+
Lexer
23+
}
24+
25+
// Lexes Markdown, highlighting a leading YAML frontmatter block before delegating to Markdown rules.
26+
func (m *markdownLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) {
27+
frontmatter, rest, ok := splitFrontmatter(text)
28+
if !ok {
29+
return m.Lexer.Tokenise(options, text)
30+
}
31+
32+
yamlLexer := Get("YAML")
33+
if yamlLexer == nil {
34+
return m.Lexer.Tokenise(options, text)
35+
}
36+
37+
yamlTokens, err := yamlLexer.Tokenise(options, frontmatter)
38+
if err != nil {
39+
return nil, err
40+
}
41+
markdownTokens, err := m.Lexer.Tokenise(options, rest)
42+
if err != nil {
43+
return nil, err
44+
}
45+
return Concaterator(yamlTokens, markdownTokens), nil
46+
}
47+
48+
// Extracts a leading YAML frontmatter block if the document starts with one.
49+
func splitFrontmatter(text string) (frontmatter string, rest string, ok bool) {
50+
if !strings.HasPrefix(text, "---\n") && !strings.HasPrefix(text, "---\r\n") {
51+
return "", text, false
52+
}
53+
54+
lineEnd := strings.IndexByte(text, '\n')
55+
if lineEnd < 0 {
56+
return "", text, false
57+
}
58+
if strings.TrimSuffix(text[:lineEnd], "\r") != "---" {
59+
return "", text, false
60+
}
61+
62+
for pos := lineEnd + 1; pos < len(text); {
63+
next := strings.IndexByte(text[pos:], '\n')
64+
if next < 0 {
65+
break
66+
}
67+
lineEnd = pos + next
68+
line := strings.TrimSuffix(text[pos:lineEnd], "\r")
69+
if line == "---" {
70+
return text[:lineEnd+1], text[lineEnd+1:], true
71+
}
72+
pos = lineEnd + 1
73+
}
74+
return "", text, false
75+
}
1776

1877
func markdownRules() Rules {
1978
return Rules{
2079
"root": {
80+
{`<!--[\w\W]*?-->`, CommentMultiline, nil},
2181
{`^(#[^#].+\n)`, ByGroups(GenericHeading), nil},
2282
{`^(#{2,6}.+\n)`, ByGroups(GenericSubheading), nil},
2383
{`^(\s*)([*-] )(\[[ xX]\])( .+\n)`, ByGroups(Text, Keyword, Keyword, UsingSelf("inline")), nil},
@@ -33,6 +93,7 @@ func markdownRules() Rules {
3393
Include("inline"),
3494
},
3595
"inline": {
96+
{`<!--[\w\W]*?-->`, CommentMultiline, nil},
3697
{`\\.`, Text, nil},
3798
{`(\s)(\*|_)((?:(?!\2).)*)(\2)((?=\W|\n))`, ByGroups(Text, GenericEmph, GenericEmph, GenericEmph, Text), nil},
3899
{`(\s)((\*\*|__).*?)\3((?=\W|\n))`, ByGroups(Text, GenericStrong, GenericStrong, Text), nil},

‎lexers/markdown_test.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package lexers
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"testing"
7+
8+
assert "github.com/alecthomas/assert/v2"
9+
"github.com/alecthomas/chroma/v2"
10+
)
11+
12+
func TestSplitFrontmatter(t *testing.T) {
13+
tests := []struct {
14+
name string
15+
input string
16+
frontmatter string
17+
rest string
18+
ok bool
19+
}{
20+
{
21+
name: "leading block",
22+
input: "---\nkey: value\n---\nbody\n",
23+
frontmatter: "---\nkey: value\n---\n",
24+
rest: "body\n",
25+
ok: true,
26+
},
27+
{
28+
name: "horizontal rule only",
29+
input: "---\nbody\n",
30+
rest: "---\nbody\n",
31+
ok: false,
32+
},
33+
{
34+
name: "mid document block",
35+
input: "body\n---\nkey: value\n---\n",
36+
rest: "body\n---\nkey: value\n---\n",
37+
ok: false,
38+
},
39+
{
40+
name: "unterminated block",
41+
input: "---\nkey: value\nbody\n",
42+
rest: "---\nkey: value\nbody\n",
43+
ok: false,
44+
},
45+
{
46+
name: "crlf block",
47+
input: "---\r\nkey: value\r\n---\r\nbody\r\n",
48+
frontmatter: "---\r\nkey: value\r\n---\r\n",
49+
rest: "body\r\n",
50+
ok: true,
51+
},
52+
}
53+
54+
for _, test := range tests {
55+
t.Run(test.name, func(t *testing.T) {
56+
frontmatter, rest, ok := splitFrontmatter(test.input)
57+
assert.Equal(t, test.ok, ok)
58+
assert.Equal(t, test.frontmatter, frontmatter)
59+
assert.Equal(t, test.rest, rest)
60+
})
61+
}
62+
}
63+
64+
func TestMarkdownFrontmatterAndCommentStress(t *testing.T) {
65+
var b strings.Builder
66+
b.WriteString("---\n")
67+
b.WriteString("title: stress test\n")
68+
b.WriteString("summary: markdown frontmatter and comment parsing\n")
69+
b.WriteString("layout: docs\n")
70+
b.WriteString("---\n\n")
71+
72+
for i := range 16 {
73+
fmt.Fprintf(&b, "Paragraph %03d <!-- comment %03d with --- and title: not frontmatter -->\n", i, i)
74+
if i%8 == 0 {
75+
fmt.Fprintf(&b, "---\nbody_key_%03d: should stay text\n---\n", i)
76+
}
77+
if i%16 == 0 {
78+
b.WriteString("---\n\n")
79+
}
80+
b.WriteString("\n")
81+
}
82+
83+
input := b.String()
84+
tokens, err := chroma.Tokenise(Markdown, nil, input)
85+
assert.NoError(t, err)
86+
assert.Equal(t, input, chroma.Stringify(tokens...))
87+
88+
var commentCount int
89+
var frontmatterNameTags int
90+
var bodyNameTags int
91+
for _, token := range tokens {
92+
assert.NotEqual(t, chroma.Error, token.Type)
93+
94+
switch {
95+
case token.Type == chroma.CommentMultiline && strings.HasPrefix(token.Value, "<!-- comment "):
96+
commentCount++
97+
case token.Type == chroma.NameTag && (token.Value == "title" || token.Value == "summary" || token.Value == "layout"):
98+
frontmatterNameTags++
99+
case token.Type == chroma.NameTag && strings.HasPrefix(token.Value, "body_key_"):
100+
bodyNameTags++
101+
}
102+
}
103+
104+
assert.Equal(t, 16, commentCount)
105+
assert.Equal(t, 3, frontmatterNameTags)
106+
assert.Zero(t, bodyNameTags)
107+
}

‎lexers/testdata/markdown.actual‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
---
2+
title: Markdown
3+
summary: Frontmatter highlighting
4+
---
5+
16
# about
27

38
<div class="html">HTML</div>
@@ -86,6 +91,20 @@ end
8691
$$;
8792
```
8893

94+
Inline HTML comment: <!-- comment -->.
95+
96+
Text before a fake frontmatter block.
97+
98+
---
99+
title: not frontmatter
100+
---
101+
102+
Text before a horizontal rule.
103+
104+
---
105+
106+
Text after a horizontal rule.
107+
89108
## MarkdownLink Test
90109
[normal link](https://google.com)
91110
[whitespace before link](https://google.com)

‎lexers/testdata/markdown.expected‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
11
[
2+
{"type":"NameNamespace","value":"---"},
3+
{"type":"TextWhitespace","value":"\n"},
4+
{"type":"NameTag","value":"title"},
5+
{"type":"Punctuation","value":":"},
6+
{"type":"TextWhitespace","value":" "},
7+
{"type":"Literal","value":"Markdown"},
8+
{"type":"TextWhitespace","value":"\n"},
9+
{"type":"NameTag","value":"summary"},
10+
{"type":"Punctuation","value":":"},
11+
{"type":"TextWhitespace","value":" "},
12+
{"type":"Literal","value":"Frontmatter highlighting"},
13+
{"type":"TextWhitespace","value":"\n"},
14+
{"type":"NameNamespace","value":"---"},
15+
{"type":"TextWhitespace","value":"\n"},
16+
{"type":"Text","value":"\n"},
217
{"type":"GenericHeading","value":"# about\n"},
318
{"type":"Text","value":"\n\u003cdiv class=\"html\"\u003eHTML\u003c/div\u003e\n\nMultiple "},
419
{"type":"GenericStrong","value":"**bold**"},
@@ -429,7 +444,9 @@
429444
{"type":"Punctuation","value":";"},
430445
{"type":"Text","value":"\n"},
431446
{"type":"LiteralString","value":"```"},
432-
{"type":"Text","value":"\n\n"},
447+
{"type":"Text","value":"\n\nInline HTML comment: "},
448+
{"type":"CommentMultiline","value":"\u003c!-- comment --\u003e"},
449+
{"type":"Text","value":".\n\nText before a fake frontmatter block.\n\n---\ntitle: not frontmatter\n---\n\nText before a horizontal rule.\n\n---\n\nText after a horizontal rule.\n\n"},
433450
{"type":"GenericSubheading","value":"## MarkdownLink Test\n"},
434451
{"type":"Text","value":"["},
435452
{"type":"NameTag","value":"normal link"},

‎styles/gruvbox-light.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
<entry type="NameException" style="noinherit #fb4934"/>
1111
<entry type="NameFunction" style="#b57614"/>
1212
<entry type="NameLabel" style="noinherit #9d0006"/>
13+
<entry type="NameNamespace" style="noinherit #79740e"/>
1314
<entry type="NameTag" style="noinherit #9d0006"/>
1415
<entry type="NameVariable" style="noinherit #3c3836"/>
1516
<entry type="LiteralString" style="noinherit #79740e"/>

‎styles/gruvbox.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
<entry type="NameFunction" style="#fabd2f"/>
1212
<entry type="NameLabel" style="noinherit #fb4934"/>
1313
<entry type="NameTag" style="noinherit #fb4934"/>
14+
<entry type="NameNamespace" style="noinherit #b8bb26"/>
1415
<entry type="NameVariable" style="noinherit #ebdbb2"/>
1516
<entry type="LiteralString" style="noinherit #b8bb26"/>
1617
<entry type="LiteralStringSymbol" style="#83a598"/>

0 commit comments

Comments
 (0)