Add helper script to generate supported langs markdown table (#1170)

ficd0 · web-flow · commit 42c2aa1c7efe · 2025-12-09T02:28:39.000+11:00
**TL;DR: added a python script that generates a "supported languages"
markdown table, making it easier to keep the table in the readme up to
date**

---

This PR adds a Python script `_tools/format_supported_langs.py` which
accepts the output of `chroma --list` over stdin and outputs a Markdown
table which can be added to the Supported Languages section of the
README. Here's how I called the script:

```sh
env -C cmd/chroma go run . | uv run _tools/format_supported_langs.py
```

I chose not to try automating edits to the README, and leave it up to
the invoker to replace the table in the readme with this script's
output. I also updated the table in the README because it's missing a
few languages.

Hopefully this can help with the maintenance burden.
diff --git a/README.md b/README.md
@@ -35,30 +35,30 @@ translators for Pygments lexers and styles.
 ## Supported languages
 
 | Prefix | Language
-| :----: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+| :----: | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 |   A    | ABAP, ABNF, ActionScript, ActionScript 3, Ada, Agda, AL, Alloy, Angular2, ANTLR, ApacheConf, APL, AppleScript, ArangoDB AQL, Arduino, ArmAsm, ATL, AutoHotkey, AutoIt, Awk
 |   B    | Ballerina, Bash, Bash Session, Batchfile, Beef, BibTeX, Bicep, BlitzBasic, BNF, BQN, Brainfuck
-|   C    | C, C#, C++, Caddyfile, Caddyfile Directives, Cap'n Proto, Cassandra CQL, Ceylon, CFEngine3, cfstatement, ChaiScript, Chapel, Cheetah, Clojure, CMake, COBOL, CoffeeScript, Common Lisp, Coq, Core, Crystal, CSS, CSV, CUE, Cython
+|   C    | C, C#, C++, C3, Caddyfile, Caddyfile Directives, Cap'n Proto, Cassandra CQL, Ceylon, CFEngine3, cfstatement, ChaiScript, Chapel, Cheetah, Clojure, CMake, COBOL, CoffeeScript, Common Lisp, Coq, Core, Crystal, CSS, CSV, CUE, Cython
 |   D    | D, Dart, Dax, Desktop file, Diff, Django/Jinja, dns, Docker, DTD, Dylan
 |   E    | EBNF, Elixir, Elm, EmacsLisp, Erlang
 |   F    | Factor, Fennel, Fish, Forth, Fortran, FortranFixed, FSharp
 |   G    | GAS, GDScript, GDScript3, Gemtext, Genshi, Genshi HTML, Genshi Text, Gherkin, Gleam, GLSL, Gnuplot, Go, Go HTML Template, Go Template, Go Text Template, GraphQL, Groff, Groovy
 |   H    | Handlebars, Hare, Haskell, Haxe, HCL, Hexdump, HLB, HLSL, HolyC, HTML, HTTP, Hy
 |   I    | Idris, Igor, INI, Io, ISCdhcpd
 |   J    | J, Janet, Java, JavaScript, JSON, JSONata, Jsonnet, Julia, Jungle
-|   K    | Kotlin
+|   K    | Kakoune, Kotlin
 |   L    | Lean4, Lighttpd configuration file, LLVM, lox, Lua
-|   M    | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, MCFunction, Meson, Metal, MiniZinc, MLIR, Modula-2, Mojo, MonkeyC, MoonScript, MorrowindScript, Myghty, MySQL
+|   M    | Makefile, Mako, markdown, Mason, Materialize SQL dialect, Mathematica, Matlab, MCFunction, Meson, Metal, MiniZinc, MLIR, Modelica, Modula-2, Mojo, MonkeyC, MoonScript, MorrowindScript, Myghty, MySQL
 |   N    | NASM, Natural, NDISASM, Newspeak, Nginx configuration file, Nim, Nix, NSIS, Nu
 |   O    | Objective-C, ObjectPascal, OCaml, Octave, Odin, OnesEnterprise, OpenEdge ABL, OpenSCAD, Org Mode
 |   P    | PacmanConf, Perl, PHP, PHTML, Pig, PkgConfig, PL/pgSQL, plaintext, Plutus Core, Pony, PostgreSQL SQL dialect, PostScript, POVRay, PowerQuery, PowerShell, Prolog, Promela, PromQL, properties, Protocol Buffer, Protocol Buffer Text Format, PRQL, PSL, Puppet, Python, Python 2
 |   Q    | QBasic, QML
-|   R    | R, Racket, Ragel, Raku, react, ReasonML, reg, Rego, reStructuredText, Rexx, RPGLE, RPMSpec, Ruby, Rust
+|   R    | R, Racket, Ragel, Raku, react, ReasonML, reg, Rego, reStructuredText, Rexx, RGBDS Assembly, Ring, RPGLE, RPMSpec, Ruby, Rust
 |   S    | SAS, Sass, Scala, Scheme, Scilab, SCSS, Sed, Sieve, Smali, Smalltalk, Smarty, SNBT, Snobol, Solidity, SourcePawn, SPARQL, SQL, SquidConf, Standard ML, stas, Stylus, Svelte, Swift, SYSTEMD, systemverilog
 |   T    | TableGen, Tal, TASM, Tcl, Tcsh, Termcap, Terminfo, Terraform, TeX, Thrift, TOML, TradingView, Transact-SQL, Turing, Turtle, Twig, TypeScript, TypoScript, TypoScriptCssData, TypoScriptHtmlData, Typst
 |   U    | ucode
 |   V    | V, V shell, Vala, VB.net, verilog, VHDL, VHS, VimL, vue
-|   W    | WDTE, WebGPU Shading Language, WebVTT, Whiley
+|   W    | WDTE, WebAssembly Text Format, WebGPU Shading Language, WebVTT, Whiley
 |   X    | XML, Xorg
 |   Y    | YAML, YANG
 |   Z    | Z80 Assembly, Zed, Zig
@@ -211,7 +211,7 @@ the following:
 ```sh
 uv run --script _tools/pygments2chroma_xml.py \
   pygments.lexers.jvm.KotlinLexer \
-  > lexers/embedded/kotlin.xml 
+  > lexers/embedded/kotlin.xml
 ```
 
 A list of all lexers available in Pygments can be found in [pygments-lexers.txt](https://github.com/alecthomas/chroma/blob/master/pygments-lexers.txt).
diff --git a/_tools/format_supported_langs.py b/_tools/format_supported_langs.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import re
+import sys
+from collections import defaultdict
+
+
+def parse_lexers(lines: list[str]) -> list[str]:
+	"""Parse the output of chroma --list and return a list of lexer names"""
+
+	lexer_name_re: re.Pattern[str] = re.compile(r"^  ([^:\s].*?)\s*$")
+	lexers: list[str] = []
+	in_lexers = False
+
+	for line in lines:
+		line = line.rstrip()
+		if line.startswith("lexers:"):
+			in_lexers = True
+			continue
+		if not in_lexers:
+			continue
+
+		# stop when we hit styles/formatters/etc
+		if line.startswith("styles:") or line.startswith("formatters:"):
+			break
+
+		match: re.Match[str] | None = lexer_name_re.match(line)
+		if match:
+			name: str | None = match.group(1)
+			if name:
+				lexers.append(name)
+	return lexers
+
+
+def group_by_prefix(lexers: list[str]) -> dict[str, list[str]]:
+	"""Given a list of lexer names, return a dictionary mapping prefixes
+	to lists of lexers that begin with that prefix"""
+	groups: defaultdict[str, list[str]] = defaultdict(list[str])
+	for name in lexers:
+		prefix: str = name[0].upper()
+		groups[prefix].append(name)
+	# sort alphabetically
+	for k in groups:
+		groups[k] = sorted(groups[k], key=lambda s: s.lower())
+	return dict(sorted(groups.items()))
+
+
+def emit_markdown(groups: dict[str, list[str]]) -> str:
+	lines: list[str] = []
+	longest = 0
+	for prefix, lexers in groups.items():
+		joined: str = ", ".join(lexers)
+		l: int = len(joined)
+		if l > longest:
+			longest: int = l
+		lines.append(f"|   {prefix}    | {joined}")
+	splitter = f"| :----: | {longest * '-'}"
+	markdown: list[str] = ["| Prefix | Language", splitter]
+	markdown.extend(lines)
+	return "\n".join(markdown)
+
+
+if __name__ == "__main__":
+	if sys.stdin.isatty():
+		print(
+			"This script parses chroma --list piped from stdin and emits a markdown table for the README"
+		)
+		print("Recommended usage (from repo root):")
+		print(
+			"env -C cmd/chroma go run . --list | uv run _tools/format_supported_langs.py"
+		)
+		exit(1)
+
+	lines: list[str] | None = sys.stdin.readlines()
+	if lines:
+		lexers: list[str] = parse_lexers(lines)
+		groups: dict[str, list[str]] = group_by_prefix(lexers)
+		print(emit_markdown(groups))
+	else:
+		exit(1)
