Pug: Improved filter tokenization (#3258)

RunDevelopment · JaKXz · web-flow · commit 0390e64420f0 · 2021-12-07T18:07:29.000+01:00
Co-authored-by: Jason Kurian &lt;JaKXz@users.noreply.github.com&gt;
diff --git a/components/prism-pug.js b/components/prism-pug.js
@@ -33,7 +33,8 @@
 				'filter-name': {
 					pattern: /^:[\w-]+/,
 					alias: 'variable'
-				}
+				},
+				'text': /\S[\s\S]*/,
 			}
 		},
 
@@ -172,7 +173,11 @@
 						pattern: /^:[\w-]+/,
 						alias: 'variable'
 					},
-					rest: Prism.languages[filter.language]
+					'text': {
+						pattern: /\S[\s\S]*/,
+						alias: [filter.language, 'language-' + filter.language],
+						inside: Prism.languages[filter.language]
+					}
 				}
 			};
 		}
diff --git a/components/prism-pug.min.js b/components/prism-pug.min.js
diff --git a/tests/languages/coffeescript+pug/coffeescript_inclusion.test b/tests/languages/coffeescript+pug/coffeescript_inclusion.test
@@ -6,14 +6,16 @@
 [
 	["filter-coffee", [
 		["filter-name", ":coffee"],
-		["string", [
-			"\"",
-			["interpolation", "#{foo}"],
-			"\""
+		["text", [
+			["string", [
+				"\"",
+				["interpolation", "#{foo}"],
+				"\""
+			]]
 		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for coffee filter (CoffeeScript) in Jade.
+Checks for coffee filter (CoffeeScript) in pug.
diff --git a/tests/languages/ejs+pug/ejs_inclusion.test b/tests/languages/ejs+pug/ejs_inclusion.test
@@ -6,19 +6,20 @@
 [
 	["filter-ejs", [
 		["filter-name", ":ejs"],
-		["language-javascript", [
-			["operator", "<"],
-			["operator", "%"],
-			["keyword", "var"],
-			" foo ",
-			["operator", "="],
-			["string", "''"],
-			["punctuation", ","],
-			" bar ",
-			["operator", "="],
-			["boolean", "false"],
-			["punctuation", ";"]
-		]],
-		["delimiter", "%>"]
+		["text", [
+			["delimiter", "<%"],
+			["language-javascript", [
+				["keyword", "var"],
+				" foo ",
+				["operator", "="],
+				["string", "''"],
+				["punctuation", ","],
+				" bar ",
+				["operator", "="],
+				["boolean", "false"],
+				["punctuation", ";"]
+			]],
+			["delimiter", "%>"]
+		]]
 	]]
 ]
diff --git a/tests/languages/handlebars+pug/handlebars_inclusion.test b/tests/languages/handlebars+pug/handlebars_inclusion.test
@@ -6,10 +6,12 @@
 [
 	["filter-handlebars", [
 		["filter-name", ":handlebars"],
-		["comment", "{{!comment}}"]
+		["text", [
+			["comment", "{{!comment}}"]
+		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for handlebars filter in Jade.
+Checks for handlebars filter in pug.
diff --git a/tests/languages/less+pug/less_inclusion.test b/tests/languages/less+pug/less_inclusion.test
@@ -6,15 +6,17 @@
 [
 	["filter-less", [
 		["filter-name", ":less"],
-		["variable", [
-			"@foo",
-			["punctuation", ":"]
-		]],
-		" #123",
-		["punctuation", ";"]
+		["text", [
+			["variable", [
+				"@foo",
+				["punctuation", ":"]
+			]],
+			" #123",
+			["punctuation", ";"]
+		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for less filter in Jade.
+Checks for less filter in pug.
diff --git a/tests/languages/livescript+pug/livescript_inclusion.test b/tests/languages/livescript+pug/livescript_inclusion.test
@@ -6,23 +6,25 @@
 [
 	["filter-livescript", [
 		["filter-name", ":livescript"],
-		["interpolated-string", [
-			["string", "\""],
-			["variable", "#foo"],
-			["string", " "],
-			["interpolation", [
-				["interpolation-punctuation", "#{"],
-				["keyword", "if"],
-				["regex", "/test/"],
-				["operator", "=="],
-				["string", "'test'"],
-				["keyword", "then"],
-				["number", "3"],
-				["keyword", "else"],
-				["number", "4"],
-				["interpolation-punctuation", "}"]
-			]],
-			["string", "\""]
+		["text", [
+			["interpolated-string", [
+				["string", "\""],
+				["variable", "#foo"],
+				["string", " "],
+				["interpolation", [
+					["interpolation-punctuation", "#{"],
+					["keyword", "if"],
+					["regex", "/test/"],
+					["operator", "=="],
+					["string", "'test'"],
+					["keyword", "then"],
+					["number", "3"],
+					["keyword", "else"],
+					["number", "4"],
+					["interpolation-punctuation", "}"]
+				]],
+				["string", "\""]
+			]]
 		]]
 	]]
 ]
diff --git a/tests/languages/markdown+pug/markdown_inclusion.test b/tests/languages/markdown+pug/markdown_inclusion.test
@@ -6,13 +6,15 @@
 [
 	["filter-markdown", [
 		["filter-name", ":markdown"],
-		["title", [
-			["punctuation", "#"],
-			" title"
+		["text", [
+			["title", [
+				["punctuation", "#"],
+				" title"
+			]]
 		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for markdown filter in Jade.
+Checks for markdown filter in pug.
diff --git a/tests/languages/pug/filter_feature.test b/tests/languages/pug/filter_feature.test
@@ -6,6 +6,6 @@
 [
 	["filter", [
 		["filter-name", ":language"],
-		"\r\n\tcode"
+		["text", "code"]
 	]]
 ]
diff --git a/tests/languages/scss+pug/scss_inclusion.test b/tests/languages/scss+pug/scss_inclusion.test
@@ -6,12 +6,14 @@
 [
 	["filter-sass", [
 		["filter-name", ":sass"],
-		["keyword", "@extend"],
-		" .foo",
-		["punctuation", ";"]
+		["text", [
+			["keyword", "@extend"],
+			" .foo",
+			["punctuation", ";"]
+		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for sass filter (Scss) in Jade.
+Checks for sass filter (Scss) in pug.
diff --git a/tests/languages/stylus+pug/stylus_inclusion.test b/tests/languages/stylus+pug/stylus_inclusion.test
@@ -6,15 +6,17 @@
 [
 	["filter-stylus", [
 		["filter-name", ":stylus"],
-		["variable-declaration", [
-			["variable", "font-size"],
-			["operator", "="],
-			["number", "14"],
-			["unit", "px"]
+		["text", [
+			["variable-declaration", [
+				["variable", "font-size"],
+				["operator", "="],
+				["number", "14"],
+				["unit", "px"]
+			]]
 		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for stylus filter in Jade.
+Checks for stylus filter in pug.
diff --git a/tests/languages/twig+pug/twig_inclusion.test b/tests/languages/twig+pug/twig_inclusion.test
@@ -6,14 +6,14 @@
 [
 	["filter-atpl", [
 		["filter-name", ":atpl"],
-
-		["punctuation", "{"],
-		["punctuation", "{"],
-		["number", "42"],
-		["delimiter", "}}"]
+		["text", [
+			["delimiter", "{{"],
+			["number", "42"],
+			["delimiter", "}}"]
+		]]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for atpl filter (Twig) in Jade.
+Checks for atpl filter (Twig) in pug.

Original file line number	Diff line number	Diff line change
`@@ -6,10 +6,12 @@`
`6`	`6`	`[`
`7`	`7`	`["filter-handlebars", [`
`8`	`8`	`["filter-name", ":handlebars"],`
`9`		`- ["comment", "{{!comment}}"]`
	`9`	`+ ["text", [`
	`10`	`+ ["comment", "{{!comment}}"]`
	`11`	`+ ]]`
`10`	`12`	`]]`
`11`	`13`	`]`
`12`	`14`
`13`	`15`	`----------------------------------------------------`
`14`	`16`
`15`		`-Checks for handlebars filter in Jade.`
	`17`	`+Checks for handlebars filter in pug.`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,6 @@`
`6`	`6`	`[`
`7`	`7`	`["filter", [`
`8`	`8`	`["filter-name", ":language"],`
`9`		`- "\r\n\tcode"`
	`9`	`+ ["text", "code"]`
`10`	`10`	`]]`
`11`	`11`	`]`