Skip to content

Commit a299df9

Browse files
MunksgaardMunksgaard
authored
Add futhark lexer (#1691)
1 parent a93ded4 commit a299df9

4 files changed

Lines changed: 366 additions & 0 deletions

File tree

‎pygments/lexers/_mapping.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@
171171
'FortranLexer': ('pygments.lexers.fortran', 'Fortran', ('fortran',), ('*.f03', '*.f90', '*.F03', '*.F90'), ('text/x-fortran',)),
172172
'FoxProLexer': ('pygments.lexers.foxpro', 'FoxPro', ('foxpro', 'vfp', 'clipper', 'xbase'), ('*.PRG', '*.prg'), ()),
173173
'FreeFemLexer': ('pygments.lexers.freefem', 'Freefem', ('freefem',), ('*.edp',), ('text/x-freefem',)),
174+
'FutharkLexer': ('pygments.lexers.futhark', 'Futhark', ('futhark',), ('*.fut',), ('text/x-futhark',)),
174175
'GAPLexer': ('pygments.lexers.algebra', 'GAP', ('gap',), ('*.g', '*.gd', '*.gi', '*.gap'), ()),
175176
'GDScriptLexer': ('pygments.lexers.gdscript', 'GDScript', ('gdscript', 'gd'), ('*.gd',), ('text/x-gdscript', 'application/x-gdscript')),
176177
'GLShaderLexer': ('pygments.lexers.graphics', 'GLSL', ('glsl',), ('*.vert', '*.frag', '*.geo'), ('text/x-glslsrc',)),

‎pygments/lexers/futhark.py‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
"""
2+
pygments.lexers.futhark
3+
~~~~~~~~~~~~~~~~~~~~~~~
4+
5+
Lexer for the Futhark language
6+
7+
:copyright: Copyright 2021 by the Pygments team, see AUTHORS.
8+
:license: BSD, see LICENSE for details.
9+
"""
10+
11+
import re
12+
13+
from pygments.lexer import RegexLexer, include, bygroups, default, words
14+
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
15+
Number, Punctuation, Error
16+
from pygments import unistring as uni
17+
18+
__all__ = ['FutharkLexer']
19+
20+
21+
line_re = re.compile('.*?\n')
22+
23+
24+
class FutharkLexer(RegexLexer):
25+
"""
26+
A Futhark lexer
27+
28+
.. versionadded:: 2.8.0
29+
"""
30+
name = 'Futhark'
31+
aliases = ['futhark']
32+
filenames = ['*.fut']
33+
mimetypes = ['text/x-futhark']
34+
35+
flags = re.MULTILINE | re.UNICODE
36+
37+
num_types = ('i8', 'i16', 'i32', 'i64', 'u8', 'u16', 'u32', 'u64', 'f32', 'f64')
38+
39+
other_types = ('bool', )
40+
41+
reserved = ('if', 'then', 'else', 'let', 'loop', 'in', 'with', 'type',
42+
'val', 'entry', 'for', 'while', 'do', 'case', 'match',
43+
'include', 'import', 'module', 'open', 'local', 'assert', '_')
44+
45+
ascii = ('NUL', 'SOH', '[SE]TX', 'EOT', 'ENQ', 'ACK',
46+
'BEL', 'BS', 'HT', 'LF', 'VT', 'FF', 'CR', 'S[OI]', 'DLE',
47+
'DC[1-4]', 'NAK', 'SYN', 'ETB', 'CAN',
48+
'EM', 'SUB', 'ESC', '[FGRU]S', 'SP', 'DEL')
49+
50+
num_postfix = r'(%s)?' % '|'.join(num_types)
51+
52+
identifier_re = '[a-zA-Z_][a-zA-Z_0-9\']*'
53+
54+
# opstart_re = '+\-\*/%=\!><\|&\^'
55+
56+
tokens = {
57+
'root': [
58+
(r'--(.*?)\n', Comment.Single),
59+
(r'\s+', Text.Whitespace),
60+
(r'\(\)', Punctuation),
61+
(r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved),
62+
(r'\b(%s)(?!\')\b' % '|'.join(num_types + other_types), Keyword.Type),
63+
64+
# Identifiers
65+
(r'#\[([a-zA-Z_\(\) ]*)\]', Comment.Preproc),
66+
(r'!?(%s\.)*%s' % (identifier_re, identifier_re), Name),
67+
68+
(r'\\', Operator),
69+
(r'[-+/%=!><|&*^][-+/%=!><|&*^.]*', Operator),
70+
(r'[][(),:;`{}]', Punctuation),
71+
72+
# Numbers
73+
(r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*_*[pP][+-]?\d(_*\d)*' + num_postfix, Number.Float),
74+
(r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*\.[\da-fA-F](_*[\da-fA-F])*'
75+
r'(_*[pP][+-]?\d(_*\d)*)?' + num_postfix, Number.Float),
76+
(r'\d(_*\d)*_*[eE][+-]?\d(_*\d)*' + num_postfix, Number.Float),
77+
(r'\d(_*\d)*\.\d(_*\d)*(_*[eE][+-]?\d(_*\d)*)?' + num_postfix, Number.Float),
78+
(r'0[bB]_*[01](_*[01])*' + num_postfix, Number.Bin),
79+
(r'0[xX]_*[\da-fA-F](_*[\da-fA-F])*' + num_postfix, Number.Hex),
80+
(r'\d(_*\d)*' + num_postfix, Number.Integer),
81+
82+
# Character/String Literals
83+
(r"'", String.Char, 'character'),
84+
(r'"', String, 'string'),
85+
# Special
86+
(r'\[[a-zA-Z_\d]*\]', Keyword.Type),
87+
(r'\(\)', Name.Builtin),
88+
],
89+
'character': [
90+
# Allows multi-chars, incorrectly.
91+
(r"[^\\']'", String.Char, '#pop'),
92+
(r"\\", String.Escape, 'escape'),
93+
("'", String.Char, '#pop'),
94+
],
95+
'string': [
96+
(r'[^\\"]+', String),
97+
(r"\\", String.Escape, 'escape'),
98+
('"', String, '#pop'),
99+
],
100+
101+
'escape': [
102+
(r'[abfnrtv"\'&\\]', String.Escape, '#pop'),
103+
(r'\^[][' + uni.Lu + r'@^_]', String.Escape, '#pop'),
104+
('|'.join(ascii), String.Escape, '#pop'),
105+
(r'o[0-7]+', String.Escape, '#pop'),
106+
(r'x[\da-fA-F]+', String.Escape, '#pop'),
107+
(r'\d+', String.Escape, '#pop'),
108+
(r'\s+\\', String.Escape, '#pop'),
109+
],
110+
}

‎tests/examplefiles/futhark/example.fut‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
module matmul (F: numeric) = {
2+
type t = F.t
3+
open F
4+
5+
local let dotprod [n] (a: [n]t) (b: [n]t): t =
6+
map2 (*) a b |> reduce (+) (i32 0)
7+
8+
9+
let matmul [n1][n2][m] (xss: [n1][m]t) (yss: [m][n2]t): [n1][n2]t =
10+
map (\xs -> map (\ys -> dotprod xs ys) <| transpose yss) xss
11+
}
12+
13+
module matmul32 = matmul f32
14+
15+
let main [n][m] (xss: [n][m]f32) (yss: [m][n]f32): *[n][n]f32 =
16+
matmul32.matmul xss yss

‎tests/examplefiles/futhark/example.fut.output‎

Lines changed: 239 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)