Skip to content

Commit 57db366

Browse files
devversiondylhunn
devversion
authored and
dylhunn
committed
refactor(migrations): framework to build batchable migrations (#57396)
Introduces a migration framework to build batchable migrations that can run in Large Scale mode against e.g. all of Google, using workers. This is the original signal input migration infrastructure extracted into a more generic framework that we can use for writing additional ones for output, signal queries etc, while making sure those are not scoped to a single `ts.Program` that limits them to per-directory execution in very large projects (e.g. G3). The migration will be updated to use this, and in 1P we will add helpers to easily integrate such migrations into a Go-based pipeline runner. PR Close #57396
1 parent 3e6ee7e commit 57db366

File tree

17 files changed

+858
-0
lines changed

17 files changed

+858
-0
lines changed

‎packages/core/schematics/utils/tsurge/BUILD.bazel‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
load("//tools:defaults.bzl", "ts_library")
2+
3+
ts_library(
4+
name = "tsurge",
5+
srcs = glob(["**/*.ts"]),
6+
visibility = [
7+
"//packages/core/schematics/utils/tsurge/test:__pkg__",
8+
],
9+
deps = [
10+
"//packages/compiler-cli",
11+
"//packages/compiler-cli/src/ngtsc/core",
12+
"//packages/compiler-cli/src/ngtsc/core:api",
13+
"//packages/compiler-cli/src/ngtsc/file_system",
14+
"//packages/compiler-cli/src/ngtsc/file_system/testing",
15+
"//packages/compiler-cli/src/ngtsc/shims",
16+
"@npm//@types/node",
17+
"@npm//magic-string",
18+
"@npm//typescript",
19+
],
20+
)

‎packages/core/schematics/utils/tsurge/beam_pipeline/read_units_blob.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import * as fs from 'fs';
10+
import * as readline from 'readline';
11+
import {TsurgeMigration} from '../migration';
12+
13+
/**
14+
* Integrating a `Tsurge` migration requires the "merging" of all
15+
* compilation unit data into a single "global migration data".
16+
*
17+
* This is achieved in a Beam pipeline by having a pipeline stage that
18+
* takes all compilation unit worker data and writing it into a single
19+
* buffer, delimited by new lines (`\n`).
20+
*
21+
* This "merged bytes files", containing all unit data, one per line, can
22+
* then be parsed by this function and fed into the migration merge logic.
23+
*
24+
* @returns All compilation unit data for the migration.
25+
*/
26+
export function readCompilationUnitBlob<UnitData, GlobalData>(
27+
_migrationForTypeSafety: TsurgeMigration<UnitData, GlobalData>,
28+
mergedUnitDataByteAbsFilePath: string,
29+
): Promise<UnitData[]> {
30+
return new Promise((resolve, reject) => {
31+
const rl = readline.createInterface({
32+
input: fs.createReadStream(mergedUnitDataByteAbsFilePath, 'utf8'),
33+
crlfDelay: Infinity,
34+
});
35+
36+
const unitData: UnitData[] = [];
37+
let failed = false;
38+
rl.on('line', (line) => {
39+
const trimmedLine = line.trim();
40+
if (trimmedLine === '') {
41+
return;
42+
}
43+
44+
try {
45+
const parsed = JSON.parse(trimmedLine) as UnitData;
46+
unitData.push(parsed);
47+
} catch (e) {
48+
failed = true;
49+
reject(new Error(`Could not parse data line: ${e} — ${trimmedLine}`));
50+
rl.close();
51+
}
52+
});
53+
54+
rl.on('close', async () => {
55+
if (!failed) {
56+
resolve(unitData);
57+
}
58+
});
59+
});
60+
}

‎packages/core/schematics/utils/tsurge/executors/analyze_exec.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import {TsurgeMigration} from '../migration';
10+
import {Serializable} from '../helpers/serializable';
11+
12+
/**
13+
* Executes the analyze phase of the given migration against
14+
* the specified TypeScript project.
15+
*
16+
* @returns the serializable migration unit data.
17+
*/
18+
export async function executeAnalyzePhase<UnitData, GlobalData>(
19+
migration: TsurgeMigration<UnitData, GlobalData>,
20+
tsconfigAbsolutePath: string,
21+
): Promise<Serializable<UnitData>> {
22+
const baseInfo = migration.createProgram(tsconfigAbsolutePath);
23+
const info = migration.prepareProgram(baseInfo);
24+
25+
return await migration.analyze(info);
26+
}

‎packages/core/schematics/utils/tsurge/executors/merge_exec.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import {Serializable} from '../helpers/serializable';
10+
import {TsurgeMigration} from '../migration';
11+
12+
/**
13+
* Executes the merge phase for the given migration against
14+
* the given set of analysis unit data.
15+
*
16+
* @returns the serializable migration global data.
17+
*/
18+
export async function executeMergePhase<UnitData, GlobalData>(
19+
migration: TsurgeMigration<UnitData, GlobalData>,
20+
units: UnitData[],
21+
): Promise<Serializable<GlobalData>> {
22+
return await migration.merge(units);
23+
}

‎packages/core/schematics/utils/tsurge/executors/migrate_exec.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import {TsurgeMigration} from '../migration';
10+
import {Replacement} from '../replacement';
11+
12+
/**
13+
* Executes the migrate phase of the given migration against
14+
* the specified TypeScript project.
15+
*
16+
* This requires the global migration data, computed by the
17+
* analysis and merge phases of the migration.
18+
*
19+
* @returns a list of text replacements to apply to disk.
20+
*/
21+
export async function executeMigratePhase<UnitData, GlobalData>(
22+
migration: TsurgeMigration<UnitData, GlobalData>,
23+
globalMetadata: GlobalData,
24+
tsconfigAbsolutePath: string,
25+
): Promise<Replacement[]> {
26+
const baseInfo = migration.createProgram(tsconfigAbsolutePath);
27+
const info = migration.prepareProgram(baseInfo);
28+
29+
return await migration.migrate(globalMetadata, info);
30+
}

‎packages/core/schematics/utils/tsurge/helpers/group_replacements.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import {AbsoluteFsPath} from '../../../../../compiler-cli/src/ngtsc/file_system';
10+
import {Replacement, TextUpdate} from '../replacement';
11+
12+
/**
13+
* Groups the given replacements per file path.
14+
*
15+
* This allows for simple execution of the replacements
16+
* against a given file. E.g. via {@link applyTextUpdates}.
17+
*/
18+
export function groupReplacementsByFile(
19+
replacements: Replacement[],
20+
): Map<AbsoluteFsPath, TextUpdate[]> {
21+
const result = new Map<AbsoluteFsPath, TextUpdate[]>();
22+
for (const {absoluteFilePath, update} of replacements) {
23+
if (!result.has(absoluteFilePath)) {
24+
result.set(absoluteFilePath, []);
25+
}
26+
result.get(absoluteFilePath)!.push(update);
27+
}
28+
return result;
29+
}

‎packages/core/schematics/utils/tsurge/helpers/ngtsc_program.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import {readConfiguration} from '../../../../../compiler-cli/src/perform_compile';
10+
import {NgCompilerOptions} from '../../../../../compiler-cli/src/ngtsc/core/api';
11+
import {
12+
FileSystem,
13+
NgtscCompilerHost,
14+
NodeJSFileSystem,
15+
setFileSystem,
16+
} from '../../../../../compiler-cli/src/ngtsc/file_system';
17+
import {NgtscProgram} from '../../../../../compiler-cli/src/ngtsc/program';
18+
import {BaseProgramInfo} from '../program_info';
19+
20+
/**
21+
* Parses the configuration of the given TypeScript project and creates
22+
* an instance of the Angular compiler for for the project.
23+
*/
24+
export function createNgtscProgram(
25+
absoluteTsconfigPath: string,
26+
fs?: FileSystem,
27+
optionOverrides: NgCompilerOptions = {},
28+
): BaseProgramInfo<NgtscProgram> {
29+
if (fs === undefined) {
30+
fs = new NodeJSFileSystem();
31+
setFileSystem(fs);
32+
}
33+
34+
const tsconfig = readConfiguration(absoluteTsconfigPath, {}, fs);
35+
36+
if (tsconfig.errors.length > 0) {
37+
throw new Error(
38+
`Tsconfig could not be parsed or is invalid:\n\n` +
39+
`${tsconfig.errors.map((e) => e.messageText)}`,
40+
);
41+
}
42+
43+
const tsHost = new NgtscCompilerHost(fs, tsconfig.options);
44+
const ngtscProgram = new NgtscProgram(
45+
tsconfig.rootNames,
46+
{
47+
...tsconfig.options,
48+
// Migrations commonly make use of TCB information.
49+
_enableTemplateTypeChecker: true,
50+
// Avoid checking libraries to speed up migrations.
51+
skipLibCheck: true,
52+
skipDefaultLibCheck: true,
53+
// Additional override options.
54+
...optionOverrides,
55+
},
56+
tsHost,
57+
);
58+
59+
return {
60+
program: ngtscProgram,
61+
userOptions: tsconfig.options,
62+
tsconfigAbsolutePath: absoluteTsconfigPath,
63+
};
64+
}

‎packages/core/schematics/utils/tsurge/helpers/serializable.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
/** Branded type indicating that the given data `T` is serializable. */
10+
export type Serializable<T> = T & {__serializable: true};
11+
12+
/** Confirms that the given data `T` is serializable. */
13+
export function confirmAsSerializable<T>(data: T): Serializable<T> {
14+
return data as Serializable<T>;
15+
}

‎packages/core/schematics/utils/tsurge/helpers/unique_id.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
/**
10+
* Helper type for creating unique branded IDs.
11+
*
12+
* Unique IDs are a fundamental piece for a `Tsurge` migration because
13+
* they allow for serializable analysis data between the stages.
14+
*
15+
* This is important to e.g. uniquely identify an Angular input across
16+
* compilation units, so that shared global data can be built via
17+
* the `merge` phase.
18+
*
19+
* E.g. a unique ID for an input may be the project-relative file path,
20+
* in combination with the name of its owning class, plus the field name.
21+
*/
22+
export type UniqueID<Name> = string & {__branded: Name};

‎packages/core/schematics/utils/tsurge/migration.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
9+
import {FileSystem} from '../../../../compiler-cli/src/ngtsc/file_system';
10+
import {NgtscProgram} from '../../../../compiler-cli/src/ngtsc/program';
11+
import assert from 'assert';
12+
import path from 'path';
13+
import ts from 'typescript';
14+
import {isShim} from '../../../../compiler-cli/src/ngtsc/shims';
15+
import {createNgtscProgram} from './helpers/ngtsc_program';
16+
import {Serializable} from './helpers/serializable';
17+
import {Replacement} from './replacement';
18+
import {BaseProgramInfo, ProgramInfo} from './program_info';
19+
20+
/**
21+
* Class defining a `Tsurge` migration.
22+
*
23+
* A tsurge migration is split into three stages:
24+
* - analyze phase
25+
* - merge phase
26+
* - migrate phase
27+
*
28+
* The motivation for such split is that migrations may be executed
29+
* on individual workers, e.g. via go/tsunami or a Beam pipeline. The
30+
* individual workers are never seeing the full project, e.g. Google3.
31+
*
32+
* The analysis phases can operate on smaller TS project units, and later
33+
* the expect the isolated unit data to be merged into some sort of global
34+
* metadata via the `merge` phase. For example, every analyze worker may
35+
* contribute to a list of TS references that are later combined.
36+
*
37+
* The migrate phase can then compute actual file updates for all individual
38+
* compilation units, leveraging the global metadata to e.g. see if there are
39+
* any references from other compilation units that may be problematic and prevent
40+
* migration of a given file.
41+
*
42+
* More details can be found in the design doc for signal input migration,
43+
* or in the testing examples.
44+
*
45+
* TODO: Link design doc.
46+
*/
47+
export abstract class TsurgeMigration<
48+
UnitAnalysisMetadata,
49+
CombinedGlobalMetadata,
50+
TsProgramType extends ts.Program | NgtscProgram = NgtscProgram,
51+
FullProgramInfo extends ProgramInfo<TsProgramType> = ProgramInfo<TsProgramType>,
52+
> {
53+
// By default, ngtsc programs are being created.
54+
createProgram(tsconfigAbsPath: string, fs?: FileSystem): BaseProgramInfo<TsProgramType> {
55+
return createNgtscProgram(tsconfigAbsPath, fs) as BaseProgramInfo<TsProgramType>;
56+
}
57+
58+
// Optional function to prepare the base `ProgramInfo` even further,
59+
// for the analyze and migrate phases. E.g. determining source files.
60+
prepareProgram(info: BaseProgramInfo<TsProgramType>): FullProgramInfo {
61+
assert(info.program instanceof NgtscProgram);
62+
63+
const userProgram = info.program.getTsProgram();
64+
const fullProgramSourceFiles = userProgram.getSourceFiles();
65+
const sourceFiles = fullProgramSourceFiles.filter(
66+
(f) =>
67+
!f.isDeclarationFile &&
68+
// Note `isShim` will work for the initial program, but for TCB programs, the shims are no longer annotated.
69+
!isShim(f) &&
70+
!f.fileName.endsWith('.ngtypecheck.ts'),
71+
);
72+
73+
const basePath = path.dirname(info.tsconfigAbsolutePath);
74+
const projectDirAbsPath = info.userOptions.rootDir ?? basePath;
75+
76+
return {
77+
...info,
78+
sourceFiles,
79+
fullProgramSourceFiles,
80+
projectDirAbsPath,
81+
} as FullProgramInfo;
82+
}
83+
84+
/** Analyzes the given TypeScript project and returns serializable compilation unit data. */
85+
abstract analyze(program: FullProgramInfo): Promise<Serializable<UnitAnalysisMetadata>>;
86+
87+
/** Merges all compilation unit data from previous analysis phases into a global metadata. */
88+
abstract merge(units: UnitAnalysisMetadata[]): Promise<Serializable<CombinedGlobalMetadata>>;
89+
90+
/**
91+
* Computes migration updates for the given TypeScript project, leveraging the global
92+
* metadata built up from all analyzed projects and their merged "unit data".
93+
*/
94+
abstract migrate(
95+
globalMetadata: CombinedGlobalMetadata,
96+
program: FullProgramInfo,
97+
): Promise<Replacement[]>;
98+
}

0 commit comments

Comments
 (0)