Skip to content

Commit 86028ad

Browse files
RunDevelopmentRunDevelopment
authored
Ruby: Improved tokenization (#3193)
1 parent 6add768 commit 86028ad

23 files changed

+1109
-525
lines changed

‎components/prism-crystal.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
number: /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/
1212
});
1313

14-
Prism.languages.insertBefore('crystal', 'string', {
14+
Prism.languages.insertBefore('crystal', 'string-literal', {
1515
attribute: {
1616
pattern: /@\[.+?\]/,
1717
alias: 'attr-name',

‎components/prism-crystal.min.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎components/prism-ruby.js‎

Lines changed: 134 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -6,133 +6,184 @@
66
*/
77
(function (Prism) {
88
Prism.languages.ruby = Prism.languages.extend('clike', {
9-
'comment': [
10-
/#.*/,
11-
{
12-
pattern: /^=begin\s[\s\S]*?^=end/m,
13-
greedy: true
14-
}
15-
],
9+
'comment': {
10+
pattern: /#.*|^=begin\s[\s\S]*?^=end/m,
11+
greedy: true
12+
},
1613
'class-name': {
17-
pattern: /(\b(?:class)\s+|\bcatch\s+\()[\w.\\]+/i,
14+
pattern: /(\b(?:class|module)\s+|\bcatch\s+\()[\w.\\]+|\b[A-Z_]\w*(?=\s*\.\s*new\b)/,
1815
lookbehind: true,
1916
inside: {
2017
'punctuation': /[.\\]/
2118
}
2219
},
23-
'keyword': /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/
20+
'keyword': /\b(?:BEGIN|END|alias|and|begin|break|case|class|def|define_method|defined|do|each|else|elsif|end|ensure|extend|for|if|in|include|module|new|next|nil|not|or|prepend|private|protected|public|raise|redo|require|rescue|retry|return|self|super|then|throw|undef|unless|until|when|while|yield)\b/,
21+
'operator': /\.{2,3}|&\.|===|<?=>|[!=]?~|(?:&&|\|\||<<|>>|\*\*|[+\-*/%<>!^&|=])=?|[?:]/,
22+
'punctuation': /[(){}[\].,;]/,
23+
});
24+
25+
Prism.languages.insertBefore('ruby', 'operator', {
26+
'double-colon': {
27+
pattern: /::/,
28+
alias: 'punctuation'
29+
},
2430
});
2531

2632
var interpolation = {
27-
pattern: /#\{[^}]+\}/,
33+
pattern: /((?:^|[^\\])(?:\\{2})*)#\{(?:[^{}]|\{[^{}]*\})*\}/,
34+
lookbehind: true,
2835
inside: {
36+
'content': {
37+
pattern: /^(#\{)[\s\S]+(?=\}$)/,
38+
lookbehind: true,
39+
inside: Prism.languages.ruby
40+
},
2941
'delimiter': {
3042
pattern: /^#\{|\}$/,
31-
alias: 'tag'
32-
},
33-
rest: Prism.languages.ruby
43+
alias: 'punctuation'
44+
}
3445
}
3546
};
3647

3748
delete Prism.languages.ruby.function;
3849

50+
var percentExpression = '(?:' + [
51+
/([^a-zA-Z0-9\s{(\[<=])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source,
52+
/\((?:[^()\\]|\\[\s\S]|\((?:[^()\\]|\\[\s\S])*\))*\)/.source,
53+
/\{(?:[^{}\\]|\\[\s\S]|\{(?:[^{}\\]|\\[\s\S])*\})*\}/.source,
54+
/\[(?:[^\[\]\\]|\\[\s\S]|\[(?:[^\[\]\\]|\\[\s\S])*\])*\]/.source,
55+
/<(?:[^<>\\]|\\[\s\S]|<(?:[^<>\\]|\\[\s\S])*>)*>/.source
56+
].join('|') + ')';
57+
58+
var symbolName = /(?:"(?:\\.|[^"\\\r\n])*"|(?:\b[a-zA-Z_]\w*|[^\s\0-\x7F]+)[?!]?|\$.)/.source;
59+
3960
Prism.languages.insertBefore('ruby', 'keyword', {
40-
'regex': [
61+
'regex-literal': [
4162
{
42-
pattern: RegExp(/%r/.source + '(?:' + [
43-
/([^a-zA-Z0-9\s{(\[<])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source,
44-
/\((?:[^()\\]|\\[\s\S])*\)/.source,
45-
// Here we need to specifically allow interpolation
46-
/\{(?:[^#{}\\]|#(?:\{[^}]+\})?|\\[\s\S])*\}/.source,
47-
/\[(?:[^\[\]\\]|\\[\s\S])*\]/.source,
48-
/<(?:[^<>\\]|\\[\s\S])*>/.source
49-
].join('|') + ')' + /[egimnosux]{0,6}/.source),
63+
pattern: RegExp(/%r/.source + percentExpression + /[egimnosux]{0,6}/.source),
5064
greedy: true,
5165
inside: {
52-
'interpolation': interpolation
66+
'interpolation': interpolation,
67+
'regex': /[\s\S]+/
5368
}
5469
},
5570
{
5671
pattern: /(^|[^/])\/(?!\/)(?:\[[^\r\n\]]+\]|\\.|[^[/\\\r\n])+\/[egimnosux]{0,6}(?=\s*(?:$|[\r\n,.;})#]))/,
5772
lookbehind: true,
5873
greedy: true,
5974
inside: {
60-
'interpolation': interpolation
75+
'interpolation': interpolation,
76+
'regex': /[\s\S]+/
6177
}
6278
}
6379
],
6480
'variable': /[@$]+[a-zA-Z_]\w*(?:[?!]|\b)/,
65-
'symbol': {
66-
pattern: /(^|[^:]):[a-zA-Z_]\w*(?:[?!]|\b)/,
67-
lookbehind: true
68-
},
81+
'symbol': [
82+
{
83+
pattern: RegExp(/(^|[^:]):/.source + symbolName),
84+
lookbehind: true,
85+
greedy: true
86+
},
87+
{
88+
pattern: RegExp(/([\r\n{(,][ \t]*)/.source + symbolName + /(?=:(?!:))/.source),
89+
lookbehind: true,
90+
greedy: true
91+
},
92+
],
6993
'method-definition': {
70-
pattern: /(\bdef\s+)[\w.]+/,
94+
pattern: /(\bdef\s+)\w+(?:\s*\.\s*\w+)?/,
7195
lookbehind: true,
7296
inside: {
73-
'function': /\w+$/,
74-
rest: Prism.languages.ruby
97+
'function': /\b\w+$/,
98+
'keyword': /^self\b/,
99+
'class-name': /^\w+/,
100+
'punctuation': /\./
75101
}
76102
}
77103
});
78104

79-
Prism.languages.insertBefore('ruby', 'number', {
80-
'builtin': /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/,
81-
'constant': /\b[A-Z]\w*(?:[?!]|\b)/
82-
});
83-
84-
Prism.languages.ruby.string = [
85-
{
86-
pattern: RegExp(/%[qQiIwWxs]?/.source + '(?:' + [
87-
/([^a-zA-Z0-9\s{(\[<])(?:(?!\1)[^\\]|\\[\s\S])*\1/.source,
88-
/\((?:[^()\\]|\\[\s\S])*\)/.source,
89-
// Here we need to specifically allow interpolation
90-
/\{(?:[^#{}\\]|#(?:\{[^}]+\})?|\\[\s\S])*\}/.source,
91-
/\[(?:[^\[\]\\]|\\[\s\S])*\]/.source,
92-
/<(?:[^<>\\]|\\[\s\S])*>/.source
93-
].join('|') + ')'),
94-
greedy: true,
95-
inside: {
96-
'interpolation': interpolation
97-
}
98-
},
99-
{
100-
pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/,
101-
greedy: true,
102-
inside: {
103-
'interpolation': interpolation
105+
Prism.languages.insertBefore('ruby', 'string', {
106+
'string-literal': [
107+
{
108+
pattern: RegExp(/%[qQiIwWs]?/.source + percentExpression),
109+
greedy: true,
110+
inside: {
111+
'interpolation': interpolation,
112+
'string': /[\s\S]+/
113+
}
114+
},
115+
{
116+
pattern: /("|')(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|(?!\1)[^\\#\r\n])*\1/,
117+
greedy: true,
118+
inside: {
119+
'interpolation': interpolation,
120+
'string': /[\s\S]+/
121+
}
122+
},
123+
{
124+
pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
125+
alias: 'heredoc-string',
126+
greedy: true,
127+
inside: {
128+
'delimiter': {
129+
pattern: /^<<[-~]?[a-z_]\w*|\b[a-z_]\w*$/i,
130+
inside: {
131+
'symbol': /\b\w+/,
132+
'punctuation': /^<<[-~]?/
133+
}
134+
},
135+
'interpolation': interpolation,
136+
'string': /[\s\S]+/
137+
}
138+
},
139+
{
140+
pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
141+
alias: 'heredoc-string',
142+
greedy: true,
143+
inside: {
144+
'delimiter': {
145+
pattern: /^<<[-~]?'[a-z_]\w*'|\b[a-z_]\w*$/i,
146+
inside: {
147+
'symbol': /\b\w+/,
148+
'punctuation': /^<<[-~]?'|'$/,
149+
}
150+
},
151+
'string': /[\s\S]+/
152+
}
104153
}
105-
},
106-
{
107-
pattern: /<<[-~]?([a-z_]\w*)[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
108-
alias: 'heredoc-string',
109-
greedy: true,
110-
inside: {
111-
'delimiter': {
112-
pattern: /^<<[-~]?[a-z_]\w*|[a-z_]\w*$/i,
113-
alias: 'symbol',
114-
inside: {
115-
'punctuation': /^<<[-~]?/
154+
],
155+
'command-literal': [
156+
{
157+
pattern: RegExp(/%x/.source + percentExpression),
158+
greedy: true,
159+
inside: {
160+
'interpolation': interpolation,
161+
'command': {
162+
pattern: /[\s\S]+/,
163+
alias: 'string'
116164
}
117-
},
118-
'interpolation': interpolation
119-
}
120-
},
121-
{
122-
pattern: /<<[-~]?'([a-z_]\w*)'[\r\n](?:.*[\r\n])*?[\t ]*\1/i,
123-
alias: 'heredoc-string',
124-
greedy: true,
125-
inside: {
126-
'delimiter': {
127-
pattern: /^<<[-~]?'[a-z_]\w*'|[a-z_]\w*$/i,
128-
alias: 'symbol',
129-
inside: {
130-
'punctuation': /^<<[-~]?'|'$/,
165+
}
166+
},
167+
{
168+
pattern: /`(?:#\{[^}]+\}|#(?!\{)|\\(?:\r\n|[\s\S])|[^\\`#\r\n])*`/,
169+
greedy: true,
170+
inside: {
171+
'interpolation': interpolation,
172+
'command': {
173+
pattern: /[\s\S]+/,
174+
alias: 'string'
131175
}
132176
}
133177
}
134-
}
135-
];
178+
]
179+
});
180+
181+
delete Prism.languages.ruby.string;
182+
183+
Prism.languages.insertBefore('ruby', 'number', {
184+
'builtin': /\b(?:Array|Bignum|Binding|Class|Continuation|Dir|Exception|FalseClass|File|Fixnum|Float|Hash|IO|Integer|MatchData|Method|Module|NilClass|Numeric|Object|Proc|Range|Regexp|Stat|String|Struct|Symbol|TMS|Thread|ThreadGroup|Time|TrueClass)\b/,
185+
'constant': /\b[A-Z][A-Z0-9_]*(?:[?!]|\b)/
186+
});
136187

137188
Prism.languages.rb = Prism.languages.ruby;
138189
}(Prism));

‎components/prism-ruby.min.js‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/languages/crystal/attribute_feature.test‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,54 @@
11
@[AlwaysInline]
22
@[CallConvention("X86_StdCall")]
3+
@[MyAnnotation(key: "value", value: 123)]
4+
@[MyAnnotation("foo", 123, false)]
35

46
----------------------------------------------------
57

68
[
79
["attribute", [
810
["delimiter", "@["],
9-
["constant", "AlwaysInline"],
11+
"AlwaysInline",
1012
["delimiter", "]"]
1113
]],
1214
["attribute", [
1315
["delimiter", "@["],
14-
["constant", "CallConvention"], ["punctuation", "("], ["string", [ "\"X86_StdCall\"" ]], ["punctuation", ")"],
16+
"CallConvention",
17+
["punctuation", "("],
18+
["string-literal", [
19+
["string", "\"X86_StdCall\""]
20+
]],
21+
["punctuation", ")"],
22+
["delimiter", "]"]
23+
]],
24+
["attribute", [
25+
["delimiter", "@["],
26+
"MyAnnotation",
27+
["punctuation", "("],
28+
["symbol", "key"],
29+
["operator", ":"],
30+
["string-literal", [
31+
["string", "\"value\""]
32+
]],
33+
["punctuation", ","],
34+
["symbol", "value"],
35+
["operator", ":"],
36+
["number", "123"],
37+
["punctuation", ")"],
38+
["delimiter", "]"]
39+
]],
40+
["attribute", [
41+
["delimiter", "@["],
42+
"MyAnnotation",
43+
["punctuation", "("],
44+
["string-literal", [
45+
["string", "\"foo\""]
46+
]],
47+
["punctuation", ","],
48+
["number", "123"],
49+
["punctuation", ","],
50+
["boolean", "false"],
51+
["punctuation", ")"],
1552
["delimiter", "]"]
1653
]]
1754
]

0 commit comments

Comments
 (0)