Skip to content

Commit a77cb72

Browse files
kevinpschaafkevinpschaaf
authored
Merge ab8b6f1 into ad361cc 
2 parents ad361cc + ab8b6f1 commit a77cb72

21 files changed

Lines changed: 690 additions & 57 deletions

File tree

‎.changeset/calm-zoos-whisper.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@lit-labs/analyzer': minor
3+
---
4+
5+
Cache Module models based on dependencies.

‎packages/labs/analyzer/.vscode/launch.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"request": "launch",
1010
"name": "Test",
1111
"skipFiles": ["<node_internals>/**"],
12-
"program": "${workspaceFolder}/../../../node_modules/.bin/uvu",
12+
"program": "${workspaceFolder}/../../../node_modules/uvu/bin.js",
1313
"args": ["test", "\\_test\\.js$"],
1414
"outFiles": ["${workspaceFolder}/**/*.js"],
1515
"console": "integratedTerminal"

‎packages/labs/analyzer/src/lib/analyzer.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
*/
66

77
import ts from 'typescript';
8-
import {Package, PackageJson, AnalyzerInterface} from './model.js';
8+
import {Package, PackageJson, AnalyzerInterface, Module} from './model.js';
99
import {AbsolutePath} from './paths.js';
1010
import {getModule} from './javascript/modules.js';
1111
export {PackageJson};
@@ -25,6 +25,9 @@ export interface AnalyzerInit {
2525
* An analyzer for Lit typescript modules.
2626
*/
2727
export class Analyzer implements AnalyzerInterface {
28+
// Cache of Module models by path; invalidated when the sourceFile
29+
// or any of its dependencies change
30+
readonly moduleCache = new Map<AbsolutePath, Module>();
2831
private readonly _getProgram: () => ts.Program;
2932
readonly fs: AnalyzerInterface['fs'];
3033
readonly path: AnalyzerInterface['path'];
@@ -45,10 +48,7 @@ export class Analyzer implements AnalyzerInterface {
4548
}
4649

4750
getModule(modulePath: AbsolutePath) {
48-
return getModule(
49-
this.program.getSourceFile(this.path.normalize(modulePath))!,
50-
this
51-
);
51+
return getModule(modulePath, this);
5252
}
5353

5454
getPackage() {
@@ -66,7 +66,7 @@ export class Analyzer implements AnalyzerInterface {
6666
...packageInfo,
6767
modules: rootFileNames.map((fileName) =>
6868
getModule(
69-
this.program.getSourceFile(this.path.normalize(fileName))!,
69+
this.path.normalize(fileName) as AbsolutePath,
7070
this,
7171
packageInfo
7272
)

‎packages/labs/analyzer/src/lib/javascript/modules.ts‎

Lines changed: 153 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -11,77 +11,190 @@
1111
*/
1212

1313
import ts from 'typescript';
14-
import {Module, AnalyzerInterface, PackageInfo} from '../model.js';
14+
import {Module, AnalyzerInterface, PackageInfo, Declaration} from '../model.js';
1515
import {
1616
isLitElement,
1717
getLitElementDeclaration,
1818
} from '../lit-element/lit-element.js';
19-
import * as path from 'path';
2019
import {getClassDeclaration} from './classes.js';
2120
import {getVariableDeclarations} from './variables.js';
2221
import {AbsolutePath, absoluteToPackage} from '../paths.js';
2322
import {getPackageInfo} from './packages.js';
23+
import {DiagnosticsError} from '../errors.js';
2424

2525
/**
26-
* Returns an analyzer `Module` model for the given ts.SourceFile.
26+
* Returns an analyzer `Module` model for the given module path.
2727
*/
2828
export const getModule = (
29-
sourceFile: ts.SourceFile,
29+
modulePath: AbsolutePath,
3030
analyzer: AnalyzerInterface,
31-
packageInfo: PackageInfo = getPackageInfo(
32-
sourceFile.fileName as AbsolutePath,
33-
analyzer
34-
)
31+
packageInfo: PackageInfo = getPackageInfo(modulePath, analyzer)
3532
) => {
36-
// Find and load the package.json associated with this module; this both gives
37-
// us the packageRoot for this module (needed for translating the source file
38-
// path to a package relative path), as well as the packageName (needed for
39-
// generating references to any symbols in this module). This will need
40-
// caching/invalidation.
33+
// Return cached module if we've parsed this sourceFile already and its
34+
// dependencies haven't changed
35+
const cachedModule = getAndValidateModuleFromCache(modulePath, analyzer);
36+
if (cachedModule !== undefined) {
37+
return cachedModule;
38+
}
39+
const sourceFile = analyzer.program.getSourceFile(
40+
analyzer.path.normalize(modulePath)
41+
);
42+
if (sourceFile === undefined) {
43+
throw new Error(`Program did not contain a source file for ${modulePath}`);
44+
}
45+
// The packageRoot for this module is needed for translating the source file
46+
// path to a package relative path, and the packageName is needed for
47+
// generating references to any symbols in this module.
4148
const {rootDir, packageJson} = packageInfo;
4249
const sourcePath = absoluteToPackage(
43-
analyzer.path.normalize(sourceFile.fileName) as AbsolutePath,
50+
analyzer.path.normalize(modulePath) as AbsolutePath,
4451
rootDir
4552
);
46-
const fullSourcePath = path.join(rootDir, sourcePath);
47-
const jsPath = fullSourcePath.endsWith('.js')
48-
? fullSourcePath
49-
: ts
50-
.getOutputFileNames(analyzer.commandLine, fullSourcePath, false)
51-
.filter((f) => f.endsWith('.js'))[0];
52-
// TODO(kschaaf): this could happen if someone imported only a .d.ts file;
53-
// we might need to handle this differently
54-
if (jsPath === undefined) {
55-
throw new Error(`Could not determine output filename for '${sourcePath}'`);
56-
}
57-
58-
const module = new Module({
59-
sourcePath,
60-
// The jsPath appears to come out of the ts API with unix
61-
// separators; since sourcePath uses OS separators, normalize
62-
// this so that all our model paths are OS-native
63-
jsPath: absoluteToPackage(
64-
analyzer.path.normalize(jsPath) as AbsolutePath,
65-
rootDir
66-
),
67-
sourceFile,
68-
packageJson,
69-
});
53+
const jsPath = absoluteToPackage(
54+
getJSPathFromSourcePath(modulePath as AbsolutePath, analyzer),
55+
rootDir
56+
);
57+
const dependencies = new Set<AbsolutePath>();
58+
const declarations: Declaration[] = [];
7059

60+
// Find and add models for declarations in the module
61+
// TODO(kschaaf): Add Variable, Function, and MixinDeclarations
7162
for (const statement of sourceFile.statements) {
7263
if (ts.isClassDeclaration(statement)) {
73-
module.declarations.push(
64+
declarations.push(
7465
isLitElement(statement, analyzer)
7566
? getLitElementDeclaration(statement, analyzer)
7667
: getClassDeclaration(statement, analyzer)
7768
);
7869
} else if (ts.isVariableStatement(statement)) {
79-
module.declarations.push(
70+
declarations.push(
8071
...statement.declarationList.declarations
8172
.map((dec) => getVariableDeclarations(dec, dec.name, analyzer))
8273
.flat()
8374
);
75+
} else if (ts.isImportDeclaration(statement)) {
76+
dependencies.add(
77+
getPathForModuleSpecifier(statement.moduleSpecifier, analyzer)
78+
);
8479
}
8580
}
81+
// Construct module and save in cache
82+
const module = new Module({
83+
sourcePath,
84+
jsPath,
85+
sourceFile,
86+
packageJson,
87+
declarations,
88+
dependencies,
89+
});
90+
analyzer.moduleCache.set(
91+
analyzer.path.normalize(sourceFile.fileName) as AbsolutePath,
92+
module
93+
);
8694
return module;
8795
};
96+
97+
/**
98+
* Returns a cached Module model for the given module path if it and all of its
99+
* dependencies' models are still valid since the model was cached. If the
100+
* cached module is out-of-date and needs to be re-created, this method returns
101+
* undefined.
102+
*/
103+
const getAndValidateModuleFromCache = (
104+
modulePath: AbsolutePath,
105+
analyzer: AnalyzerInterface
106+
): Module | undefined => {
107+
const module = analyzer.moduleCache.get(modulePath);
108+
// A cached module is only valid if the source file that was used has not
109+
// changed in the current program, and if all of its dependencies are still
110+
// valid
111+
if (module !== undefined) {
112+
if (
113+
module.sourceFile === analyzer.program.getSourceFile(modulePath) &&
114+
depsAreValid(module, analyzer)
115+
) {
116+
return module;
117+
}
118+
analyzer.moduleCache.delete(modulePath);
119+
}
120+
return undefined;
121+
};
122+
123+
/**
124+
* Returns true if all dependencies of the module are still valid.
125+
*/
126+
const depsAreValid = (module: Module, analyzer: AnalyzerInterface) =>
127+
Array.from(module.dependencies).every((path) => depIsValid(path, analyzer));
128+
129+
/**
130+
* Returns true if the given dependency is valid, meaning that if it has a
131+
* cached model, the model is still valid. Dependencies that don't yet have a
132+
* cached model are considered valid.
133+
*/
134+
const depIsValid = (modulePath: AbsolutePath, analyzer: AnalyzerInterface) => {
135+
if (analyzer.moduleCache.has(modulePath)) {
136+
// If a dep has a model, it is valid only if its deps are valid
137+
return Boolean(getAndValidateModuleFromCache(modulePath, analyzer));
138+
} else {
139+
// Deps that don't have a cached model are considered valid
140+
return true;
141+
}
142+
};
143+
144+
/**
145+
* For a given source file, return its associated JS file.
146+
*
147+
* For a JS source file, these will be the same thing. For a TS file, we use the
148+
* TS API to determine where the associated JS will be output based on tsconfig
149+
* settings.
150+
*/
151+
const getJSPathFromSourcePath = (
152+
sourcePath: AbsolutePath,
153+
analyzer: AnalyzerInterface
154+
) => {
155+
sourcePath = analyzer.path.normalize(sourcePath) as AbsolutePath;
156+
// If the source file was already JS, just return that
157+
if (sourcePath.endsWith('js')) {
158+
return sourcePath;
159+
}
160+
// Use the TS API to determine where the associated JS will be output based
161+
// on tsconfig settings.
162+
const outputPath = ts
163+
.getOutputFileNames(analyzer.commandLine, sourcePath, false)
164+
.filter((f) => f.endsWith('.js'))[0];
165+
// TODO(kschaaf): this could happen if someone imported only a .d.ts file;
166+
// we might need to handle this differently
167+
if (outputPath === undefined) {
168+
throw new Error(`Could not determine output filename for '${sourcePath}'`);
169+
}
170+
// The filename appears to come out of the ts API with
171+
// unix separators; since sourcePath uses OS separators, normalize this so
172+
// that all our model paths are OS-native
173+
return analyzer.path.normalize(outputPath) as AbsolutePath;
174+
};
175+
176+
/**
177+
* Resolve a module specifier to an absolute path on disk.
178+
*/
179+
const getPathForModuleSpecifier = (
180+
moduleSpecifier: ts.Expression,
181+
analyzer: AnalyzerInterface
182+
): AbsolutePath => {
183+
const specifier = moduleSpecifier.getText().slice(1, -1);
184+
let resolvedPath = ts.resolveModuleName(
185+
specifier,
186+
moduleSpecifier.getSourceFile().fileName,
187+
analyzer.commandLine.options,
188+
analyzer.fs
189+
).resolvedModule?.resolvedFileName;
190+
if (resolvedPath === undefined) {
191+
throw new DiagnosticsError(
192+
moduleSpecifier,
193+
`Could not resolve specifier to filesystem path.`
194+
);
195+
}
196+
if (!analyzer.fs.useCaseSensitiveFileNames) {
197+
resolvedPath = resolvedPath.toLowerCase();
198+
}
199+
return analyzer.path.normalize(resolvedPath) as AbsolutePath;
200+
};

‎packages/labs/analyzer/src/lib/javascript/packages.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ export const getPackageRootForModulePath = (
1919
const {fs, path} = analyzer;
2020
let searchDir = modulePath as string;
2121
const root = path.parse(searchDir).root;
22-
while (searchDir !== root) {
23-
if (fs.fileExists(path.join(searchDir, 'package.json'))) {
24-
return searchDir as AbsolutePath;
22+
while (!fs.fileExists(path.join(searchDir, 'package.json'))) {
23+
if (searchDir === root) {
24+
throw new Error(`No package.json found for module path ${modulePath}`);
2525
}
2626
searchDir = path.dirname(searchDir);
2727
}
28-
throw new Error(`No package.json found for module path ${modulePath}`);
28+
return searchDir as AbsolutePath;
2929
};
3030

3131
/**

‎packages/labs/analyzer/src/lib/model.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ export interface ModuleInit {
7676
sourcePath: PackagePath;
7777
jsPath: PackagePath;
7878
packageJson: PackageJson;
79+
declarations: Declaration[];
80+
dependencies: Set<AbsolutePath>;
7981
}
8082

8183
export class Module {
@@ -94,14 +96,26 @@ export class Module {
9496
* project this will be the same as `sourcePath`.
9597
*/
9698
readonly jsPath: PackagePath;
97-
readonly declarations: Array<Declaration> = [];
99+
/**
100+
* A list of all Declaration models in this module.
101+
*/
102+
readonly declarations: Array<Declaration>;
103+
/**
104+
* A set of all dependencies of this module
105+
*/
106+
readonly dependencies: Set<AbsolutePath>;
107+
/**
108+
* The package.json contents for the package containing this module.
109+
*/
98110
readonly packageJson: PackageJson;
99111

100112
constructor(init: ModuleInit) {
101113
this.sourceFile = init.sourceFile;
102114
this.sourcePath = init.sourcePath;
103115
this.jsPath = init.jsPath;
104116
this.packageJson = init.packageJson;
117+
this.declarations = init.declarations;
118+
this.dependencies = init.dependencies;
105119
}
106120
}
107121

@@ -301,6 +315,7 @@ export const getImportsStringForReferences = (references: Reference[]) => {
301315
};
302316

303317
export interface AnalyzerInterface {
318+
moduleCache: Map<AbsolutePath, Module>;
304319
program: ts.Program;
305320
commandLine: ts.ParsedCommandLine;
306321
fs: Pick<

‎packages/labs/analyzer/src/lib/references.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import ts from 'typescript';
88
import {DiagnosticsError} from './errors.js';
99
import {AnalyzerInterface, Reference} from './model.js';
1010
import {getModule} from './javascript/modules.js';
11+
import {AbsolutePath} from './paths.js';
1112

1213
const npmModule = /^(?<package>(@\w+\/\w+)|\w+)\/?(?<module>.*)$/;
1314

@@ -108,7 +109,10 @@ export function getReferenceForSymbol(
108109
if (moduleSpecifier[0] === '.') {
109110
// Relative import from this package: use the current package and
110111
// module path relative to this module
111-
const module = getModule(location.getSourceFile(), analyzer);
112+
const module = getModule(
113+
location.getSourceFile().fileName as AbsolutePath,
114+
analyzer
115+
);
112116
refPackage = module.packageJson.name;
113117
refModule = path.join(path.dirname(module.jsPath), moduleSpecifier);
114118
} else if (moduleSpecifier[0] === '/') {
@@ -137,7 +141,10 @@ export function getReferenceForSymbol(
137141
});
138142
} else {
139143
// Declared in this file: use the current package and module
140-
const module = getModule(location.getSourceFile(), analyzer);
144+
const module = getModule(
145+
location.getSourceFile().fileName as AbsolutePath,
146+
analyzer
147+
);
141148
return new Reference({
142149
name: symbolName,
143150
package: module.packageJson.name,

‎packages/labs/analyzer/src/test/analyzer_test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
* SPDX-License-Identifier: BSD-3-Clause
55
*/
66

7+
import 'source-map-support/register.js';
78
import {suite} from 'uvu';
89
// eslint-disable-next-line import/extensions
910
import * as assert from 'uvu/assert';

0 commit comments

Comments
 (0)