Skip to content

Commit ce43442

Browse files
bad-anticsbad-antics
authored
Add Lateralus lexer (#1249)
Adds a lexer for **Lateralus**, a statically typed, pipeline-oriented programming language with algebraic data types, pattern matching, and first-class effect capabilities. ## Language info - Homepage: https://lateralus.dev - Reference implementation: https://github.com/bad-antics/lateralus-lang (MIT licensed) - File extension: `.ltl` - Aliases: `lateralus`, `ltl` ## What's included - `lexers/embedded/lateralus.xml` — XML lexer config covering keywords, builtin types and functions, attribute decorators (`@memo`), capability annotations (`#[caps(io, net)]`), nested block comments, raw/byte/interpolated strings, numeric literals with type suffixes (`42u64`, `3.14f32`, `0xFF_i32`), module paths (`foo::bar`), ADT variants, and the signature pipeline operator `|>`. - `lexers/testdata/lateralus.actual` — real example from the Lateralus reference implementation's `examples/` directory (MIT licensed). - `lexers/testdata/lateralus.expected` — golden output generated via `RECORD=true go test`. ## Tests - `go test ./lexers/ -run TestLexers/Lateralus` → PASS - `go test ./lexers/ -run 'TestCompileAllRegexes|TestGlobs|TestAliases'` → PASS - Sample produces zero `Error` tokens. ## Sample license The `fibonacci.ltl` sample is copied verbatim from the Lateralus reference implementation under MIT; happy to relicense for inclusion if preferred.
1 parent 2b00673 commit ce43442

3 files changed

Lines changed: 369 additions & 0 deletions

File tree

‎lexers/embedded/lateralus.xml‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
<lexer>
2+
<config>
3+
<name>Lateralus</name>
4+
<alias>lateralus</alias>
5+
<alias>ltl</alias>
6+
<filename>*.ltl</filename>
7+
<mime_type>text/x-lateralus</mime_type>
8+
<ensure_nl>true</ensure_nl>
9+
</config>
10+
<rules>
11+
<state name="root">
12+
<!-- Whitespace -->
13+
<rule pattern="\s+">
14+
<token type="TextWhitespace"/>
15+
</rule>
16+
<!-- Comments -->
17+
<rule pattern="///[^\n]*">
18+
<token type="CommentSpecial"/>
19+
</rule>
20+
<rule pattern="//[^\n]*">
21+
<token type="CommentSingle"/>
22+
</rule>
23+
<rule pattern="/\*">
24+
<token type="CommentMultiline"/>
25+
<push state="block_comment"/>
26+
</rule>
27+
<!-- Attribute decorators: @memo, @doc("...") -->
28+
<rule pattern="@[A-Za-z_][A-Za-z0-9_]*">
29+
<token type="NameDecorator"/>
30+
</rule>
31+
<!-- Capability annotations: #[caps(io, net)] -->
32+
<rule pattern="#\[">
33+
<token type="NameDecorator"/>
34+
<push state="attribute"/>
35+
</rule>
36+
<!-- Control flow / declaration keywords -->
37+
<rule pattern="(fn|let|mut|match|if|else|elif|while|for|in|return|break|continue|import|export|module|pub|priv|struct|enum|impl|trait|where|type|const|static|async|await|spawn|guard|defer|use|as|self|Self|super|yield|do)\b">
38+
<token type="Keyword"/>
39+
</rule>
40+
<!-- Builtin types -->
41+
<rule pattern="(int|i8|i16|i32|i64|i128|uint|u8|u16|u32|u64|u128|float|f32|f64|bool|str|char|bytes|any|never|list|map|set|tuple|Option|Result|Some|None|Ok|Err)\b">
42+
<token type="KeywordType"/>
43+
</rule>
44+
<!-- Boolean / null literals -->
45+
<rule pattern="(true|false|null)\b">
46+
<token type="KeywordConstant"/>
47+
</rule>
48+
<!-- Builtin functions -->
49+
<rule pattern="(print|println|eprint|eprintln|format|panic|assert|assert_eq|todo|unimplemented|unreachable|len|range|map|filter|reduce|fold|zip|enumerate|sort|sorted|reverse|sum|min|max)\b">
50+
<token type="NameBuiltin"/>
51+
</rule>
52+
<!-- Function definition -->
53+
<rule pattern="(fn)(\s+)([A-Za-z_][A-Za-z0-9_]*)">
54+
<bygroups>
55+
<token type="Keyword"/>
56+
<token type="TextWhitespace"/>
57+
<token type="NameFunction"/>
58+
</bygroups>
59+
</rule>
60+
<!-- Type / struct / enum / trait / impl definition -->
61+
<rule pattern="(struct|enum|trait|type|impl)(\s+)([A-Z][A-Za-z0-9_]*)">
62+
<bygroups>
63+
<token type="Keyword"/>
64+
<token type="TextWhitespace"/>
65+
<token type="NameClass"/>
66+
</bygroups>
67+
</rule>
68+
<!-- Module path: foo::bar -->
69+
<rule pattern="([A-Za-z_][A-Za-z0-9_]*)(::)">
70+
<bygroups>
71+
<token type="NameNamespace"/>
72+
<token type="Punctuation"/>
73+
</bygroups>
74+
</rule>
75+
<!-- ADT variant: UppercaseIdentifier -->
76+
<rule pattern="[A-Z][A-Za-z0-9_]*">
77+
<token type="NameClass"/>
78+
</rule>
79+
<!-- Raw strings: r"...", r#"..."# -->
80+
<rule pattern="r#&#34;.*?&#34;#">
81+
<token type="LiteralString"/>
82+
</rule>
83+
<rule pattern="r&#34;[^&#34;\\]*(?:\\.[^&#34;\\]*)*&#34;">
84+
<token type="LiteralString"/>
85+
</rule>
86+
<!-- Byte strings -->
87+
<rule pattern="b&#34;">
88+
<token type="LiteralString"/>
89+
<push state="string"/>
90+
</rule>
91+
<!-- Regular / interpolated strings -->
92+
<rule pattern="&#34;">
93+
<token type="LiteralString"/>
94+
<push state="string"/>
95+
</rule>
96+
<!-- Char literals -->
97+
<rule pattern="&#39;(?:\\.|[^&#39;\\])&#39;">
98+
<token type="LiteralStringChar"/>
99+
</rule>
100+
<!-- Numeric literals -->
101+
<rule pattern="[0-9][0-9_]*\.[0-9][0-9_]*(?:[eE][-+]?[0-9][0-9_]*)?(?:_?f(32|64))?">
102+
<token type="LiteralNumberFloat"/>
103+
</rule>
104+
<rule pattern="0x[0-9a-fA-F][0-9a-fA-F_]*(?:_?[iu](8|16|32|64|128))?">
105+
<token type="LiteralNumberHex"/>
106+
</rule>
107+
<rule pattern="0o[0-7][0-7_]*(?:_?[iu](8|16|32|64|128))?">
108+
<token type="LiteralNumberOct"/>
109+
</rule>
110+
<rule pattern="0b[01][01_]*(?:_?[iu](8|16|32|64|128))?">
111+
<token type="LiteralNumberBin"/>
112+
</rule>
113+
<rule pattern="[0-9][0-9_]*(?:_?[iu](8|16|32|64|128))?">
114+
<token type="LiteralNumberInteger"/>
115+
</rule>
116+
<!-- Pipeline operator (signature feature) -->
117+
<rule pattern="\|&gt;">
118+
<token type="Operator"/>
119+
</rule>
120+
<!-- Other operators -->
121+
<rule pattern="(==|!=|&lt;=|&gt;=|-&gt;|=&gt;|&amp;&amp;|\|\||&lt;&lt;|&gt;&gt;|::|\.\.=?|\?\?|[+\-*/%&lt;&gt;!=&amp;|\^~?])">
122+
<token type="Operator"/>
123+
</rule>
124+
<!-- Punctuation -->
125+
<rule pattern="[{}()\[\];,.:]">
126+
<token type="Punctuation"/>
127+
</rule>
128+
<!-- Identifiers -->
129+
<rule pattern="[A-Za-z_][A-Za-z0-9_]*">
130+
<token type="Name"/>
131+
</rule>
132+
</state>
133+
<state name="block_comment">
134+
<rule pattern="[^/*]+">
135+
<token type="CommentMultiline"/>
136+
</rule>
137+
<rule pattern="/\*">
138+
<token type="CommentMultiline"/>
139+
<push/>
140+
</rule>
141+
<rule pattern="\*/">
142+
<token type="CommentMultiline"/>
143+
<pop depth="1"/>
144+
</rule>
145+
<rule pattern="[/*]">
146+
<token type="CommentMultiline"/>
147+
</rule>
148+
</state>
149+
<state name="string">
150+
<rule pattern="\\[nrt&#39;&#34;\\0]">
151+
<token type="LiteralStringEscape"/>
152+
</rule>
153+
<rule pattern="\\x[0-9a-fA-F]{2}">
154+
<token type="LiteralStringEscape"/>
155+
</rule>
156+
<rule pattern="\\u\{[0-9a-fA-F]+\}">
157+
<token type="LiteralStringEscape"/>
158+
</rule>
159+
<rule pattern="\{[^}]*\}">
160+
<token type="LiteralStringInterpol"/>
161+
</rule>
162+
<rule pattern="[^&#34;\\{}]+">
163+
<token type="LiteralString"/>
164+
</rule>
165+
<rule pattern="&#34;">
166+
<token type="LiteralString"/>
167+
<pop depth="1"/>
168+
</rule>
169+
</state>
170+
<state name="attribute">
171+
<rule pattern="[^\[\]]+">
172+
<token type="NameDecorator"/>
173+
</rule>
174+
<rule pattern="\[">
175+
<token type="NameDecorator"/>
176+
<push/>
177+
</rule>
178+
<rule pattern="\]">
179+
<token type="NameDecorator"/>
180+
<pop depth="1"/>
181+
</rule>
182+
</state>
183+
</rules>
184+
</lexer>

‎lexers/testdata/lateralus.actual‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// examples/fibonacci.ltl - Fibonacci demonstration
2+
3+
module examples.fibonacci
4+
5+
import io
6+
7+
fn fib(n: int) -> int {
8+
if n <= 0 { return 0 }
9+
if n == 1 { return 1 }
10+
return fib(n - 1) + fib(n - 2)
11+
}
12+
13+
fn main() {
14+
io.println("Fibonacci sequence:")
15+
let mut i = 0
16+
while i <= 15 {
17+
io.println("fib(" + i as str + ") = " + fib(i) as str)
18+
i += 1
19+
}
20+
}
21+
22+
main()

‎lexers/testdata/lateralus.expected‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
[
2+
{"type":"CommentSingle","value":"// examples/fibonacci.ltl - Fibonacci demonstration"},
3+
{"type":"TextWhitespace","value":"\n\n"},
4+
{"type":"Keyword","value":"module"},
5+
{"type":"TextWhitespace","value":" "},
6+
{"type":"Name","value":"examples"},
7+
{"type":"Punctuation","value":"."},
8+
{"type":"Name","value":"fibonacci"},
9+
{"type":"TextWhitespace","value":"\n\n"},
10+
{"type":"Keyword","value":"import"},
11+
{"type":"TextWhitespace","value":" "},
12+
{"type":"Name","value":"io"},
13+
{"type":"TextWhitespace","value":"\n\n"},
14+
{"type":"Keyword","value":"fn"},
15+
{"type":"TextWhitespace","value":" "},
16+
{"type":"Name","value":"fib"},
17+
{"type":"Punctuation","value":"("},
18+
{"type":"Name","value":"n"},
19+
{"type":"Punctuation","value":":"},
20+
{"type":"TextWhitespace","value":" "},
21+
{"type":"KeywordType","value":"int"},
22+
{"type":"Punctuation","value":")"},
23+
{"type":"TextWhitespace","value":" "},
24+
{"type":"Operator","value":"-\u003e"},
25+
{"type":"TextWhitespace","value":" "},
26+
{"type":"KeywordType","value":"int"},
27+
{"type":"TextWhitespace","value":" "},
28+
{"type":"Punctuation","value":"{"},
29+
{"type":"TextWhitespace","value":"\n "},
30+
{"type":"Keyword","value":"if"},
31+
{"type":"TextWhitespace","value":" "},
32+
{"type":"Name","value":"n"},
33+
{"type":"TextWhitespace","value":" "},
34+
{"type":"Operator","value":"\u003c="},
35+
{"type":"TextWhitespace","value":" "},
36+
{"type":"LiteralNumberInteger","value":"0"},
37+
{"type":"TextWhitespace","value":" "},
38+
{"type":"Punctuation","value":"{"},
39+
{"type":"TextWhitespace","value":" "},
40+
{"type":"Keyword","value":"return"},
41+
{"type":"TextWhitespace","value":" "},
42+
{"type":"LiteralNumberInteger","value":"0"},
43+
{"type":"TextWhitespace","value":" "},
44+
{"type":"Punctuation","value":"}"},
45+
{"type":"TextWhitespace","value":"\n "},
46+
{"type":"Keyword","value":"if"},
47+
{"type":"TextWhitespace","value":" "},
48+
{"type":"Name","value":"n"},
49+
{"type":"TextWhitespace","value":" "},
50+
{"type":"Operator","value":"=="},
51+
{"type":"TextWhitespace","value":" "},
52+
{"type":"LiteralNumberInteger","value":"1"},
53+
{"type":"TextWhitespace","value":" "},
54+
{"type":"Punctuation","value":"{"},
55+
{"type":"TextWhitespace","value":" "},
56+
{"type":"Keyword","value":"return"},
57+
{"type":"TextWhitespace","value":" "},
58+
{"type":"LiteralNumberInteger","value":"1"},
59+
{"type":"TextWhitespace","value":" "},
60+
{"type":"Punctuation","value":"}"},
61+
{"type":"TextWhitespace","value":"\n "},
62+
{"type":"Keyword","value":"return"},
63+
{"type":"TextWhitespace","value":" "},
64+
{"type":"Name","value":"fib"},
65+
{"type":"Punctuation","value":"("},
66+
{"type":"Name","value":"n"},
67+
{"type":"TextWhitespace","value":" "},
68+
{"type":"Operator","value":"-"},
69+
{"type":"TextWhitespace","value":" "},
70+
{"type":"LiteralNumberInteger","value":"1"},
71+
{"type":"Punctuation","value":")"},
72+
{"type":"TextWhitespace","value":" "},
73+
{"type":"Operator","value":"+"},
74+
{"type":"TextWhitespace","value":" "},
75+
{"type":"Name","value":"fib"},
76+
{"type":"Punctuation","value":"("},
77+
{"type":"Name","value":"n"},
78+
{"type":"TextWhitespace","value":" "},
79+
{"type":"Operator","value":"-"},
80+
{"type":"TextWhitespace","value":" "},
81+
{"type":"LiteralNumberInteger","value":"2"},
82+
{"type":"Punctuation","value":")"},
83+
{"type":"TextWhitespace","value":"\n"},
84+
{"type":"Punctuation","value":"}"},
85+
{"type":"TextWhitespace","value":"\n\n"},
86+
{"type":"Keyword","value":"fn"},
87+
{"type":"TextWhitespace","value":" "},
88+
{"type":"Name","value":"main"},
89+
{"type":"Punctuation","value":"()"},
90+
{"type":"TextWhitespace","value":" "},
91+
{"type":"Punctuation","value":"{"},
92+
{"type":"TextWhitespace","value":"\n "},
93+
{"type":"Name","value":"io"},
94+
{"type":"Punctuation","value":"."},
95+
{"type":"NameBuiltin","value":"println"},
96+
{"type":"Punctuation","value":"("},
97+
{"type":"LiteralString","value":"\"Fibonacci sequence:\""},
98+
{"type":"Punctuation","value":")"},
99+
{"type":"TextWhitespace","value":"\n "},
100+
{"type":"Keyword","value":"let"},
101+
{"type":"TextWhitespace","value":" "},
102+
{"type":"Keyword","value":"mut"},
103+
{"type":"TextWhitespace","value":" "},
104+
{"type":"Name","value":"i"},
105+
{"type":"TextWhitespace","value":" "},
106+
{"type":"Operator","value":"="},
107+
{"type":"TextWhitespace","value":" "},
108+
{"type":"LiteralNumberInteger","value":"0"},
109+
{"type":"TextWhitespace","value":"\n "},
110+
{"type":"Keyword","value":"while"},
111+
{"type":"TextWhitespace","value":" "},
112+
{"type":"Name","value":"i"},
113+
{"type":"TextWhitespace","value":" "},
114+
{"type":"Operator","value":"\u003c="},
115+
{"type":"TextWhitespace","value":" "},
116+
{"type":"LiteralNumberInteger","value":"15"},
117+
{"type":"TextWhitespace","value":" "},
118+
{"type":"Punctuation","value":"{"},
119+
{"type":"TextWhitespace","value":"\n "},
120+
{"type":"Name","value":"io"},
121+
{"type":"Punctuation","value":"."},
122+
{"type":"NameBuiltin","value":"println"},
123+
{"type":"Punctuation","value":"("},
124+
{"type":"LiteralString","value":"\"fib(\""},
125+
{"type":"TextWhitespace","value":" "},
126+
{"type":"Operator","value":"+"},
127+
{"type":"TextWhitespace","value":" "},
128+
{"type":"Name","value":"i"},
129+
{"type":"TextWhitespace","value":" "},
130+
{"type":"Keyword","value":"as"},
131+
{"type":"TextWhitespace","value":" "},
132+
{"type":"KeywordType","value":"str"},
133+
{"type":"TextWhitespace","value":" "},
134+
{"type":"Operator","value":"+"},
135+
{"type":"TextWhitespace","value":" "},
136+
{"type":"LiteralString","value":"\") = \""},
137+
{"type":"TextWhitespace","value":" "},
138+
{"type":"Operator","value":"+"},
139+
{"type":"TextWhitespace","value":" "},
140+
{"type":"Name","value":"fib"},
141+
{"type":"Punctuation","value":"("},
142+
{"type":"Name","value":"i"},
143+
{"type":"Punctuation","value":")"},
144+
{"type":"TextWhitespace","value":" "},
145+
{"type":"Keyword","value":"as"},
146+
{"type":"TextWhitespace","value":" "},
147+
{"type":"KeywordType","value":"str"},
148+
{"type":"Punctuation","value":")"},
149+
{"type":"TextWhitespace","value":"\n "},
150+
{"type":"Name","value":"i"},
151+
{"type":"TextWhitespace","value":" "},
152+
{"type":"Operator","value":"+="},
153+
{"type":"TextWhitespace","value":" "},
154+
{"type":"LiteralNumberInteger","value":"1"},
155+
{"type":"TextWhitespace","value":"\n "},
156+
{"type":"Punctuation","value":"}"},
157+
{"type":"TextWhitespace","value":"\n"},
158+
{"type":"Punctuation","value":"}"},
159+
{"type":"TextWhitespace","value":"\n\n"},
160+
{"type":"Name","value":"main"},
161+
{"type":"Punctuation","value":"()"},
162+
{"type":"TextWhitespace","value":"\n"}
163+
]

0 commit comments

Comments
 (0)