Skip to content

Commit 5333e28

Browse files
RunDevelopmentRunDevelopment
authored
Added regex coverage (#3138)
1 parent 2e834c8 commit 5333e28

File tree

8 files changed

+371
-73
lines changed

8 files changed

+371
-73
lines changed

‎.github/workflows/test.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,16 @@ jobs:
5757
node-version: 14.x
5858
- run: npm ci
5959
- run: npm run lint:ci
60+
61+
coverage:
62+
63+
runs-on: ubuntu-latest
64+
65+
steps:
66+
- uses: actions/checkout@v2
67+
- name: Use Node.js 14.x
68+
uses: actions/setup-node@v1
69+
with:
70+
node-version: 14.x
71+
- run: npm ci
72+
- run: npm run regex-coverage

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"lint": "eslint . --cache",
1515
"lint:fix": "npm run lint -- --fix",
1616
"lint:ci": "eslint . --max-warnings 0",
17+
"regex-coverage": "mocha tests/coverage.js",
1718
"test:aliases": "mocha tests/aliases-test.js",
1819
"test:core": "mocha tests/core/**/*.js",
1920
"test:dependencies": "mocha tests/dependencies-test.js",

‎tests/coverage.js‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
'use strict';
2+
3+
const TestDiscovery = require('./helper/test-discovery');
4+
const TestCase = require('./helper/test-case');
5+
const PrismLoader = require('./helper/prism-loader');
6+
const { BFS, BFSPathToPrismTokenPath } = require('./helper/util');
7+
const { assert } = require('chai');
8+
const components = require('../components.json');
9+
const ALL_LANGUAGES = [...Object.keys(components.languages).filter(k => k !== 'meta')];
10+
11+
12+
describe('Pattern test coverage', function () {
13+
/**
14+
* @type {Map<string, PatternData>}
15+
* @typedef PatternData
16+
* @property {RegExp} pattern
17+
* @property {string} language
18+
* @property {Set<string>} from
19+
* @property {RegExpExecArray[]} matches
20+
*/
21+
const patterns = new Map();
22+
23+
/**
24+
* @param {string | string[]} languages
25+
* @returns {import("./helper/prism-loader").Prism}
26+
*/
27+
function createInstance(languages) {
28+
const Prism = PrismLoader.createInstance(languages);
29+
30+
BFS(Prism.languages, (path, object) => {
31+
const { key, value } = path[path.length - 1];
32+
const tokenPath = BFSPathToPrismTokenPath(path);
33+
34+
if (Object.prototype.toString.call(value) == '[object RegExp]') {
35+
const regex = makeGlobal(value);
36+
object[key] = regex;
37+
38+
const patternKey = String(regex);
39+
let data = patterns.get(patternKey);
40+
if (!data) {
41+
data = {
42+
pattern: regex,
43+
language: path[1].key,
44+
from: new Set([tokenPath]),
45+
matches: []
46+
};
47+
patterns.set(patternKey, data);
48+
} else {
49+
data.from.add(tokenPath);
50+
}
51+
52+
regex.exec = string => {
53+
let match = RegExp.prototype.exec.call(regex, string);
54+
if (match) {
55+
data.matches.push(match);
56+
}
57+
return match;
58+
};
59+
}
60+
});
61+
62+
return Prism;
63+
}
64+
65+
describe('Register all patterns', function () {
66+
it('all', function () {
67+
this.slow(10 * 1000);
68+
// This will cause ALL regexes of Prism to be registered in the patterns map.
69+
// (Languages that don't have any tests can't be caught otherwise.)
70+
createInstance(ALL_LANGUAGES);
71+
});
72+
});
73+
74+
describe('Run all language tests', function () {
75+
// define tests for all tests in all languages in the test suite
76+
for (const [languageIdentifier, files] of TestDiscovery.loadAllTests()) {
77+
it(languageIdentifier, function () {
78+
this.timeout(10 * 1000);
79+
80+
for (const filePath of files) {
81+
try {
82+
TestCase.run({
83+
languageIdentifier,
84+
filePath,
85+
updateMode: 'none',
86+
createInstance
87+
});
88+
} catch (error) {
89+
// we don't case about whether the test succeeds,
90+
// we just want to gather usage data
91+
}
92+
}
93+
});
94+
}
95+
});
96+
97+
describe('Coverage', function () {
98+
for (const language of ALL_LANGUAGES) {
99+
describe(language, function () {
100+
it(`- should cover all patterns`, function () {
101+
const untested = getAllOf(language).filter(d => d.matches.length === 0);
102+
if (untested.length === 0) {
103+
return;
104+
}
105+
106+
const problems = untested.map(data => {
107+
return formatProblem(data, [
108+
'This pattern is completely untested. Add test files that match this pattern.'
109+
]);
110+
});
111+
112+
assert.fail([
113+
`${problems.length} pattern(s) are untested:\n`
114+
+ 'You can learn more about writing tests at https://prismjs.com/test-suite.html#writing-tests',
115+
...problems
116+
].join('\n\n'));
117+
});
118+
119+
it(`- should exhaustively cover all keywords in keyword lists`, function () {
120+
const problems = [];
121+
122+
for (const data of getAllOf(language)) {
123+
if (data.matches.length === 0) {
124+
// don't report the same pattern twice
125+
continue;
126+
}
127+
128+
const keywords = getKeywordList(data.pattern);
129+
if (!keywords) {
130+
continue;
131+
}
132+
const keywordCount = keywords.size;
133+
134+
data.matches.forEach(([m]) => {
135+
if (data.pattern.ignoreCase) {
136+
m = m.toUpperCase();
137+
}
138+
keywords.delete(m);
139+
});
140+
141+
if (keywords.size > 0) {
142+
problems.push(formatProblem(data, [
143+
`Add test files to test all keywords. The following keywords (${keywords.size}/${keywordCount}) are untested:`,
144+
...[...keywords].map(k => ` ${k}`)
145+
]));
146+
}
147+
}
148+
149+
if (problems.length === 0) {
150+
return;
151+
}
152+
153+
assert.fail([
154+
`${problems.length} keyword list(s) are not exhaustively tested:\n`
155+
+ 'You can learn more about writing tests at https://prismjs.com/test-suite.html#writing-tests',
156+
...problems
157+
].join('\n\n'));
158+
});
159+
});
160+
}
161+
});
162+
163+
/**
164+
* @param {string} language
165+
* @returns {PatternData[]}
166+
*/
167+
function getAllOf(language) {
168+
return [...patterns.values()].filter(d => d.language === language);
169+
}
170+
171+
/**
172+
* @param {string} string
173+
* @param {number} maxLength
174+
* @returns {string}
175+
*/
176+
function short(string, maxLength) {
177+
if (string.length > maxLength) {
178+
return string.slice(0, maxLength - 1) + '…';
179+
} else {
180+
return string;
181+
}
182+
}
183+
184+
/**
185+
* If the given pattern string describes a keyword list, all keyword will be returned. Otherwise, `null` will be
186+
* returned.
187+
*
188+
* @param {RegExp} pattern
189+
* @returns {Set<string> | null}
190+
*/
191+
function getKeywordList(pattern) {
192+
// Right now, only keyword lists of the form /\b(?:foo|bar)\b/ are supported.
193+
// In the future, we might want to convert these regexes to NFAs and iterate all words to cover more complex
194+
// keyword lists and even operator and punctuation lists.
195+
196+
let source = pattern.source.replace(/^\\b|\\b$/g, '');
197+
if (source.startsWith('(?:') && source.endsWith(')')) {
198+
source = source.slice('(?:'.length, source.length - ')'.length);
199+
}
200+
201+
if (/^\w+(?:\|\w+)*$/.test(source)) {
202+
if (pattern.ignoreCase) {
203+
source = source.toUpperCase();
204+
}
205+
return new Set(source.split(/\|/g));
206+
} else {
207+
return null;
208+
}
209+
}
210+
211+
/**
212+
* @param {Iterable<string>} occurrences
213+
* @returns {{ origin: string; otherOccurrences: string[] }}
214+
*/
215+
function splitOccurrences(occurrences) {
216+
const all = [...occurrences];
217+
return {
218+
origin: all[0],
219+
otherOccurrences: all.slice(1),
220+
};
221+
}
222+
223+
/**
224+
* @param {PatternData} data
225+
* @param {string[]} messageLines
226+
* @returns {string}
227+
*/
228+
function formatProblem(data, messageLines) {
229+
const { origin, otherOccurrences } = splitOccurrences(data.from);
230+
231+
const lines = [
232+
`${origin}:`,
233+
short(String(data.pattern), 100),
234+
'',
235+
...messageLines,
236+
];
237+
238+
if (otherOccurrences.length) {
239+
lines.push(
240+
'',
241+
'Other occurrences of this pattern:',
242+
...otherOccurrences.map(o => `- ${o}`)
243+
);
244+
}
245+
246+
return lines.join('\n ');
247+
}
248+
});
249+
250+
/**
251+
* @param {RegExp} regex
252+
* @returns {RegExp}
253+
*/
254+
function makeGlobal(regex) {
255+
if (regex.global) {
256+
return regex;
257+
} else {
258+
return RegExp(regex.source, regex.flags + 'g');
259+
}
260+
}

‎tests/helper/test-case.js‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
'use strict';
22

33
const fs = require('fs');
4+
const path = require('path');
45
const { assert } = require('chai');
56
const Prettier = require('prettier');
67
const PrismLoader = require('./prism-loader');
@@ -11,6 +12,12 @@ const TokenStreamTransformer = require('./token-stream-transformer');
1112
* @typedef {import("../../components/prism-core.js")} Prism
1213
*/
1314

15+
/**
16+
* @param {string[]} languages
17+
* @returns {Prism}
18+
*/
19+
const defaultCreateInstance = (languages) => PrismLoader.createInstance(languages);
20+
1421
/**
1522
* Handles parsing and printing of a test case file.
1623
*
@@ -297,6 +304,29 @@ class HighlightHTMLRunner {
297304
module.exports = {
298305
TestCaseFile,
299306

307+
/**
308+
* Runs the given test file and asserts the result.
309+
*
310+
* This function will determine what kind of test files the given file is and call the appropriate method to run the
311+
* test.
312+
*
313+
* @param {RunOptions} options
314+
* @returns {void}
315+
*
316+
* @typedef RunOptions
317+
* @property {string} languageIdentifier
318+
* @property {string} filePath
319+
* @property {"none" | "insert" | "update"} updateMode
320+
* @property {(languages: string[]) => Prism} [createInstance]
321+
*/
322+
run(options) {
323+
if (path.extname(options.filePath) === '.test') {
324+
this.runTestCase(options.languageIdentifier, options.filePath, options.updateMode, options.createInstance);
325+
} else {
326+
this.runTestsWithHooks(options.languageIdentifier, require(options.filePath), options.createInstance);
327+
}
328+
},
329+
300330
/**
301331
* Runs the given test case file and asserts the result
302332
*
@@ -312,27 +342,31 @@ module.exports = {
312342
* @param {string} languageIdentifier
313343
* @param {string} filePath
314344
* @param {"none" | "insert" | "update"} updateMode
345+
* @param {(languages: string[]) => Prism} [createInstance]
315346
*/
316-
runTestCase(languageIdentifier, filePath, updateMode) {
347+
runTestCase(languageIdentifier, filePath, updateMode, createInstance = defaultCreateInstance) {
348+
let runner;
317349
if (/\.html\.test$/i.test(filePath)) {
318-
this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new HighlightHTMLRunner());
350+
runner = new HighlightHTMLRunner();
319351
} else {
320-
this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new TokenizeJSONRunner());
352+
runner = new TokenizeJSONRunner();
321353
}
354+
this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, runner, createInstance);
322355
},
323356

324357
/**
325358
* @param {string} languageIdentifier
326359
* @param {string} filePath
327360
* @param {"none" | "insert" | "update"} updateMode
328361
* @param {Runner<T>} runner
362+
* @param {(languages: string[]) => Prism} createInstance
329363
* @template T
330364
*/
331-
runTestCaseWithRunner(languageIdentifier, filePath, updateMode, runner) {
365+
runTestCaseWithRunner(languageIdentifier, filePath, updateMode, runner, createInstance) {
332366
const testCase = TestCaseFile.readFromFile(filePath);
333367
const usedLanguages = this.parseLanguageNames(languageIdentifier);
334368

335-
const Prism = PrismLoader.createInstance(usedLanguages.languages);
369+
const Prism = createInstance(usedLanguages.languages);
336370

337371
// the first language is the main language to highlight
338372
const actualValue = runner.run(Prism, testCase.code, usedLanguages.mainLanguage);

0 commit comments

Comments
 (0)