Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ Other contributors, listed alphabetically, are:
* Michael Ficarra -- CPSA lexer
* James H. Fisher -- PostScript lexer
* William S. Fulton -- SWIG lexer
* John Gabriele -- Janet lexer
* Carlos Galdino -- Elixir and Elixir Console lexers
* Michael Galloy -- IDL lexer
* Naveen Garg -- Autohotkey lexer
Expand Down
1 change: 1 addition & 0 deletions CHANGES
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ Version 2.7.0
* Devicetree (PR#1434)
* F* (PR#1409)
* GDScript (PR#1457)
* Janet (PR#1519)
* Pointless (PR#1494)
* PromQL (PR#1506)
* PsySH (PR#1438)
Expand Down
1 change: 1 addition & 0 deletions doc/languages.rst
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ Programming languages
* `Igor Pro <https://www.wavemetrics.com/products/igorpro/programming>`_
* `Io <http://iolanguage.com/>`_
* `Jags <http://mcmc-jags.sourceforge.net/>`_
* `Janet <https://janet-lang.org/>`_
* `Java <https://www.oracle.com/java/>`_
* `JavaScript <https://en.wikipedia.org/wiki/JavaScript>`_
* `Jasmin <http://jasmin.sourceforge.net/>`_
Expand Down
1 change: 1 addition & 0 deletions pygments/lexers/_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@
'IsabelleLexer': ('pygments.lexers.theorem', 'Isabelle', ('isabelle',), ('*.thy',), ('text/x-isabelle',)),
'JLexer': ('pygments.lexers.j', 'J', ('j',), ('*.ijs',), ('text/x-j',)),
'JagsLexer': ('pygments.lexers.modeling', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()),
'JanetLexer': ('pygments.lexers.lisp', 'Janet', ('janet',), ('*.janet',), ('text/x-janet', 'application/x-janet')),
'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()),
'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)),
'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')),
Expand Down
147 changes: 146 additions & 1 deletion pygments/lexers/lisp.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

__all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer',
'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer',
'XtlangLexer', 'FennelLexer']
'XtlangLexer', 'FennelLexer', 'JanetLexer']


class SchemeLexer(RegexLexer):
Expand Down Expand Up @@ -2697,3 +2697,148 @@ class FennelLexer(RegexLexer):
(r'#', Punctuation),
]
}


class JanetLexer(RegexLexer):
"""Lexer for the `Janet programming language <https://janet-lang.org/>`_.

.. versionadded:: 2.7
"""
name = "Janet"
aliases = ["janet"]
filenames = ["*.janet"]
mimetypes = ['text/x-janet', 'application/x-janet']

special_forms = (
'break', 'def', 'do', 'fn', 'if', 'quote', 'quasiquote', 'splice',
'set', 'unquote', 'var', 'while'
)

declarations = (
'def-', 'defn', 'defn-', 'defmacro', 'defmacro-',
)

builtins = (
'%', '%=', '*', '*=', '+', '++', '+=', '-', '--', '-=',
'->', '->>', '-?>', '-?>>', '/', '/=', '<', '<=', '=', '>', '>=',
'abstract?', 'accumulate', 'accumulate2', 'all', 'all-bindings',
'all-dynamics', 'and', 'apply', 'array', 'array/concat', 'array/ensure',
'array/fill', 'array/insert', 'array/new', 'array/new-filled',
'array/peek', 'array/pop', 'array/push', 'array/remove', 'array/slice',
'array?', 'as->', 'as?->', 'asm', 'assert', 'bad-compile', 'bad-parse', 'band',
'blshift', 'bnot', 'boolean?', 'bor', 'brshift', 'brushift', 'buffer', 'buffer/bit',
'buffer/bit-clear', 'buffer/bit-set', 'buffer/bit-toggle',
'buffer/blit', 'buffer/clear', 'buffer/fill', 'buffer/format',
'buffer/new', 'buffer/new-filled', 'buffer/popn', 'buffer/push-byte',
'buffer/push-string', 'buffer/push-word', 'buffer/slice', 'buffer?',
'bxor', 'bytes?', 'case', 'cfunction?', 'chr', 'cli-main', 'comment', 'comp',
'compare', 'compare=', 'compare<', 'compare<=', 'compare>', 'compare>=',
'compile', 'complement', 'comptime', 'cond', 'coro', 'count', 'debug',
'debug/arg-stack', 'debug/break', 'debug/fbreak', 'debug/lineage',
'debug/stack', 'debug/stacktrace', 'debug/step', 'debug/unbreak',
'debug/unfbreak', 'debugger-env', 'dec', 'deep-not=', 'deep=', 'default',
'default-peg-grammar', 'defer',
'defglobal', 'describe', 'dictionary?', 'disasm', 'distinct', 'doc', 'doc*',
'doc-format', 'dofile', 'drop', 'drop-until', 'drop-while', 'dyn', 'each', 'eachk',
'eachp', 'eachy', 'edefer', 'eflush', 'empty?', 'env-lookup', 'eprin', 'eprinf',
'eprint', 'eprintf', 'error', 'errorf', 'eval', 'eval-string', 'even?', 'every?',
'extreme', 'false?', 'fiber/can-resume?', 'fiber/current', 'fiber/getenv',
'fiber/maxstack', 'fiber/new', 'fiber/root', 'fiber/setenv',
'fiber/setmaxstack', 'fiber/status', 'fiber?', 'file/close', 'file/flush',
'file/open', 'file/popen', 'file/read', 'file/seek', 'file/temp',
'file/write', 'filter', 'find', 'find-index', 'first', 'flatten', 'flatten-into',
'flush', 'for', 'forv', 'freeze', 'frequencies', 'function?', 'gccollect',
'gcinterval', 'gcsetinterval', 'generate', 'gensym', 'get', 'get-in', 'getline',
'hash', 'idempotent?', 'identity', 'import', 'import*', 'if-let', 'if-not',
'if-with', 'in', 'inc', 'indexed?', 'int/s64', 'int/u64', 'int?', 'interleave',
'interpose', 'invert', 'janet/build', 'janet/config-bits', 'janet/version',
'juxt', 'juxt*', 'keep', 'keys', 'keyword', 'keyword?', 'kvs', 'label', 'last', 'length',
'let', 'load-image', 'load-image-dict', 'loop', 'macex', 'macex1', 'make-env',
'make-image', 'make-image-dict', 'map', 'mapcat', 'marshal', 'math/-inf',
'math/abs', 'math/acos', 'math/acosh', 'math/asin', 'math/asinh', 'math/atan',
'math/atan2', 'math/atanh', 'math/cbrt', 'math/ceil', 'math/cos', 'math/cosh',
'math/e', 'math/erf', 'math/erfc', 'math/exp', 'math/exp2', 'math/expm1',
'math/floor', 'math/gamma', 'math/hypot', 'math/inf', 'math/log',
'math/log10', 'math/log1p', 'math/log2', 'math/next', 'math/pi', 'math/pow',
'math/random', 'math/rng', 'math/rng-buffer', 'math/rng-int',
'math/rng-uniform', 'math/round', 'math/seedrandom', 'math/sin',
'math/sinh', 'math/sqrt', 'math/tan', 'math/tanh', 'math/trunc', 'match', 'max',
'mean', 'merge', 'merge-into', 'min', 'mod', 'module/add-paths', 'module/cache',
'module/expand-path', 'module/find', 'module/loaders', 'module/loading',
'module/paths', 'nan?', 'nat?', 'native', 'neg?', 'net/chunk', 'net/close',
'net/connect', 'net/read', 'net/server', 'net/write', 'next', 'nil?', 'not', 'not=',
'number?', 'odd?', 'one?', 'or', 'os/arch', 'os/cd', 'os/chmod', 'os/clock',
'os/cryptorand', 'os/cwd', 'os/date', 'os/dir', 'os/environ', 'os/execute',
'os/exit', 'os/getenv', 'os/link', 'os/lstat', 'os/mkdir', 'os/mktime',
'os/perm-int', 'os/perm-string', 'os/readlink', 'os/realpath', 'os/rename',
'os/rm', 'os/rmdir', 'os/setenv', 'os/shell', 'os/sleep', 'os/stat',
'os/symlink', 'os/time', 'os/touch', 'os/umask', 'os/which', 'pairs', 'parse',
'parser/byte', 'parser/clone', 'parser/consume', 'parser/eof',
'parser/error', 'parser/flush', 'parser/has-more', 'parser/insert',
'parser/new', 'parser/produce', 'parser/state', 'parser/status',
'parser/where', 'partial', 'partition', 'peg/compile', 'peg/match', 'pos?',
'postwalk', 'pp', 'prewalk', 'prin', 'prinf', 'print', 'printf', 'product', 'prompt',
'propagate', 'protect', 'put', 'put-in', 'quit', 'range', 'reduce', 'reduce2',
'repeat', 'repl', 'require', 'resume', 'return', 'reverse', 'reversed', 'root-env',
'run-context', 'scan-number', 'seq', 'setdyn', 'shortfn', 'signal', 'slice',
'slurp', 'some', 'sort', 'sort-by', 'sorted', 'sorted-by', 'spit', 'stderr', 'stdin',
'stdout', 'string', 'string/ascii-lower', 'string/ascii-upper',
'string/bytes', 'string/check-set', 'string/find', 'string/find-all',
'string/format', 'string/from-bytes', 'string/has-prefix?',
'string/has-suffix?', 'string/join', 'string/repeat', 'string/replace',
'string/replace-all', 'string/reverse', 'string/slice', 'string/split',
'string/trim', 'string/triml', 'string/trimr', 'string?', 'struct', 'struct?',
'sum', 'symbol', 'symbol?', 'table', 'table/clone', 'table/getproto',
'table/new', 'table/rawget', 'table/setproto', 'table/to-struct', 'table?',
'take', 'take-until', 'take-while', 'tarray/buffer', 'tarray/copy-bytes',
'tarray/length', 'tarray/new', 'tarray/properties', 'tarray/slice',
'tarray/swap-bytes', 'thread/close', 'thread/current', 'thread/new',
'thread/receive', 'thread/send', 'trace', 'tracev', 'true?', 'truthy?', 'try',
'tuple', 'tuple/brackets', 'tuple/setmap', 'tuple/slice',
'tuple/sourcemap', 'tuple/type', 'tuple?', 'type', 'unless', 'unmarshal',
'untrace', 'update', 'update-in', 'use', 'values', 'var-', 'varfn', 'varglobal',
'walk', 'walk-ind', 'walk-dict', 'when', 'when-let', 'when-with', 'with',
'with-dyns', 'with-syms', 'with-vars', 'yield', 'zero?', 'zipcoll'
)

# Like Fennel's (no leading digits), and added ;,.\'
valid_name = r'[a-zA-Z_!$%&*+/:<=>?@^~|;,\'\.-][\w!$%&*+/:<=>?@^~|;,\'\.-]*'

tokens = {
'root': [
# comment
(r'#.*$', Comment.Single),

(r'\s+', Text),

# number
(r'-?\d+\.\d+', Number.Float),
(r'-?\d+', Number.Integer),
(r'0[xX][a-fA-F0-9]+', Number.Hex),

# double-quoted string
(r'@?"(\\\\|\\"|[^"])*"', String),
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is going to result in catastrophic backtracking (similar for the case below.) Please do two things to make sure we won't suffer from this:

  • Add a test case with a long string with backslashes in a string. That should take a looong time to execute, potentially timeout.
  • Change it to [^\\"] to avoid the backtracking (same below), the test should pass just fine now.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi Anteru. Sorry for the delay. Thank you for investigating this. Can you please tell me where to add this test case? I'm not familiar with adding tests to pygments, and also don't see a tests/test_lisp.py file.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just create a new file then :) The way to write a test is to use the testcase formatter (see also the docs), here's an example of the exact same bug just in the JsonLexer which you can use as a blueprint: gerner@ec77778

# backtick-quoted string
(r'@?(`+)(\\\\|\\`|[^`])*\1', String),

# symbol
(r"'" + valid_name, String.Symbol),
# keyword
(r":" + valid_name, String.Symbol),

(words(special_forms, suffix=' '), Keyword),
(words(declarations, suffix=' '), Keyword.Declaration),
(words(builtins, suffix=' '), Name.Builtin),

# the remaining functions
(r'(?<=\()' + valid_name, Name.Function),

# regular identifiers
(valid_name, Name.Variable),

# paired delimiters
(r'(\(|\))', Punctuation),
(r'(\[|\])', Punctuation),
(r'(\{|\})', Punctuation),
],
}
50 changes: 50 additions & 0 deletions tests/examplefiles/example.janet
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Just some shenanigans here to check syntax highlighting.
(print "Hello, Janet!")
(def x 3)
(def y 4.5)
(def s "hello")
(def s2 @"hi there")
(def s3 @`chit chat`)

(var a [:a :b :c])
(var a2 @[:hi-o :hey-o :bye-o])

(var t {:ayy45 "this" :bee23 "that" :sea67 "other"})
(var t2 @{:a 1 :b 2 :c 3})

(eachp [k v] t
(print k " --> " v))

(print 't2)
(print (math/sin (/ math/pi 4)))

(var sum 0)
(loop [i :range [0 10]]
(+= sum i))
(print sum) # prints 45

(defn mult-by-2
``Docstring.

Quite a docstring.

And here it ends.``
[n]
(* n 2))

(print (mult-by-2 21) " ***")

(defn fully-charged? [n] true)
(defn foo->bar [n] "1000")

(if (fully-charged? x)
(do
(print (foo->bar 12))
(print "=====")))

(let [a (filter even?
(map (fn [x]
(* x x))
(range 10)))] # => @[0 4 16 36 64]
(each item a
(print item)))