Skip to content

Commit 8d24a91

Browse files
kormidejosephperrott
kormide
authored and
josephperrott
committed
build(bazel): refactor aio example e2es to fix windows performance
Use the same config flag to enable local vs npm deps as aio.
1 parent a36034a commit 8d24a91

File tree

27 files changed

+190
-1084
lines changed

27 files changed

+190
-1084
lines changed

‎WORKSPACE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,27 @@ yarn_install(
136136
yarn_lock = "//aio:yarn.lock",
137137
)
138138

139+
yarn_install(
140+
name = "aio_example_deps",
141+
data = [
142+
YARN_LABEL,
143+
"//:.yarnrc",
144+
],
145+
# Disabled because, when False, yarn_install preserves the node_modules folder
146+
# with bin symlinks in the external repository. This is needed to link the shared
147+
# set of deps for example e2es.
148+
exports_directories_only = False,
149+
manual_build_file_contents = """\
150+
filegroup(
151+
name = "node_modules_with_bins",
152+
srcs = ["node_modules", "node_modules/.bin"],
153+
)
154+
""",
155+
package_json = "//aio/tools/examples/shared:package.json",
156+
yarn = YARN_LABEL,
157+
yarn_lock = "//aio/tools/examples/shared:yarn.lock",
158+
)
159+
139160
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
140161

141162
aspect_bazel_lib_dependencies()

‎aio/content/examples/examples.bzl‎

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ load("@build_bazel_rules_nodejs//:index.bzl", "npm_package_bin")
22
load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
33
load("//aio/tools:defaults.bzl", "nodejs_test")
44
load("//:yarn.bzl", "YARN_LABEL")
5+
load("//:packages.bzl", "ALL_PACKAGES", "to_package_label")
56

67
# This map controls which examples are included and whether or not to generate
78
# a stackblitz live examples and zip archives. Keys are the example name, and values
@@ -177,36 +178,43 @@ def docs_example(name, test = True, test_tags = []):
177178
)
178179

179180
if test:
180-
# These node_modules deps are symlinked into each example. These tree
181-
# artifact folder names must still be "node_modules" despite the symlink
182-
# being named node_modules. Otherwise, some deps will fail to resolve.
183-
node_modules_deps = {
184-
"local": "//aio/tools/examples/shared:local/node_modules",
185-
"npm": "//aio/tools/examples/shared:node_modules",
186-
}
181+
EXAMPLE_DEPS_WORKSPACE_NAME = "aio_example_deps"
187182

188-
for [node_modules_source, node_modules_label] in node_modules_deps.items():
189-
nodejs_test(
190-
name = "e2e_%s" % node_modules_source,
191-
data = [
192-
":%s" % name,
193-
YARN_LABEL,
194-
node_modules_label,
195-
"@aio_npm//@angular/dev-infra-private/bazel/browsers/chromium",
196-
],
197-
args = [
198-
"$(rootpath :%s)" % name,
199-
"$(rootpath %s)" % node_modules_label,
200-
"$(rootpath %s)" % YARN_LABEL,
201-
],
202-
configuration_env_vars = ["NG_BUILD_CACHE"],
203-
entry_point = "//aio/tools/examples:run-example-e2e",
204-
env = {
205-
"CHROME_BIN": "$(CHROMIUM)",
206-
"CHROMEDRIVER_BIN": "$(CHROMEDRIVER)",
207-
},
208-
toolchains = [
209-
"@aio_npm//@angular/dev-infra-private/bazel/browsers/chromium:toolchain_alias",
210-
],
211-
tags = test_tags,
212-
)
183+
LOCAL_PACKAGE_DEPS = [to_package_label(dep) for dep in ALL_PACKAGES]
184+
185+
# Local package deps are passed as args to the test script in the form "@package/name#path/to/package"
186+
# for the script's convenience.
187+
LOCAL_PACKAGE_ARGS = ["%s#$(rootpath %s)" % (dep, to_package_label(dep)) for dep in ALL_PACKAGES]
188+
189+
nodejs_test(
190+
name = "e2e",
191+
data = [
192+
":%s" % name,
193+
YARN_LABEL,
194+
"@aio_npm//@angular/dev-infra-private/bazel/browsers/chromium",
195+
"//aio/tools/examples:run-example-e2e",
196+
# We install the whole node modules for runtime deps of e2e tests
197+
"@{workspace}//:node_modules_with_bins".format(workspace = EXAMPLE_DEPS_WORKSPACE_NAME),
198+
] + select({
199+
"//aio:aio_local_deps": LOCAL_PACKAGE_DEPS,
200+
"//conditions:default": [],
201+
}),
202+
args = [
203+
"$(rootpath :%s)" % name,
204+
"$(rootpath %s)" % YARN_LABEL,
205+
EXAMPLE_DEPS_WORKSPACE_NAME,
206+
] + select({
207+
"//aio:aio_local_deps": LOCAL_PACKAGE_ARGS,
208+
"//conditions:default": [],
209+
}),
210+
configuration_env_vars = ["NG_BUILD_CACHE"],
211+
entry_point = "//aio/tools/examples:run-example-e2e",
212+
env = {
213+
"CHROME_BIN": "$(CHROMIUM)",
214+
"CHROMEDRIVER_BIN": "$(CHROMEDRIVER)",
215+
},
216+
toolchains = [
217+
"@aio_npm//@angular/dev-infra-private/bazel/browsers/chromium:toolchain_alias",
218+
],
219+
tags = test_tags,
220+
)

‎aio/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,8 @@
117117
"@bazel/bazelisk": "^1.7.5",
118118
"@bazel/buildozer": "^5.1.0",
119119
"@bazel/jasmine": "^5.4.1",
120-
"@bazel/typescript": "5.3.1",
121120
"@bazel/runfiles": "5.4.2",
121+
"@bazel/typescript": "5.3.1",
122122
"@types/jasmine": "~4.3.0",
123123
"@types/lunr": "^2.3.3",
124124
"@types/node": "^12.7.9",

‎aio/tools/esm-loader/esm-loader.mjs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,12 @@ export async function resolve(specifier, context, defaultResolve) {
1717
return defaultResolve(specifier, context, defaultResolve);
1818
}
1919

20-
const nodeModules = path.resolve('external', process.env.NODE_MODULES_WORKSPACE_NAME, 'node_modules');
20+
let nodeModules;
21+
if (isBazelRunOrTestAction()) {
22+
nodeModules = path.resolve('../', process.env.NODE_MODULES_WORKSPACE_NAME, 'node_modules');
23+
} else {
24+
nodeModules = path.resolve('external', process.env.NODE_MODULES_WORKSPACE_NAME, 'node_modules');
25+
}
2126

2227
const packageImport = parsePackageImport(specifier);
2328
const pathToNodeModule = path.join(nodeModules, packageImport.packageName);
@@ -54,3 +59,7 @@ function resolvePackageLocalFilepath(packageImport, packageJson) {
5459

5560
return packageImport.pathInPackage || packageJson.module || packageJson.main || 'index.js';
5661
}
62+
63+
function isBazelRunOrTestAction() {
64+
return process.env.TEST_WORKSPACE || process.env.BUILD_WORKSPACE_DIRECTORY;
65+
}

‎aio/tools/examples/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ js_library(
6565
deps = [
6666
"@aio_npm//@bazel/runfiles",
6767
"@aio_npm//canonical-path",
68+
"@aio_npm//cjson",
6869
"@aio_npm//cross-spawn",
6970
"@aio_npm//fs-extra",
7071
"@aio_npm//globby",

‎aio/tools/examples/run-example-e2e.mjs‎

Lines changed: 102 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import path from 'canonical-path';
22
import {spawn} from 'cross-spawn';
33
import fs from 'fs-extra';
4-
import {sync as globbySync} from 'globby';
4+
import {globbySync} from 'globby';
5+
import jsonc from 'cjson';
56
import os from 'os';
67
import shelljs from 'shelljs';
78
import treeKill from 'tree-kill';
@@ -20,27 +21,42 @@ process.env.CHROMEDRIVER_BIN = path.resolve(process.env.CHROMEDRIVER_BIN);
2021
const {argv} = yargs(hideBin(process.argv));
2122

2223
const EXAMPLE_PATH = path.resolve(argv._[0]);
23-
const NODE_MODULES_PATH = path.resolve(argv._[1]);
2424
const NODE = process.execPath;
25-
const VENDORED_YARN = path.resolve(argv._[2]);
25+
const VENDORED_YARN = path.resolve(argv._[1]);
26+
const EXAMPLE_DEPS_WORKSPACE_NAME = argv._[2];
27+
const LOCAL_PACKAGES = argv._.slice(3).reduce((pkgs, pkgNameAndPath) => {
28+
const [pkgName, pkgPath] = pkgNameAndPath.split('#');
29+
pkgs[pkgName] = path.resolve(pkgPath);
30+
return pkgs;
31+
}, {});
32+
2633
const SJS_SPEC_FILENAME = 'e2e-spec.ts';
2734
const CLI_SPEC_FILENAME = 'e2e/src/app.e2e-spec.ts';
2835
const EXAMPLE_CONFIG_FILENAME = 'example-config.json';
2936
const MAX_NO_OUTPUT_TIMEOUT = 1000 * 60 * 5; // 5 minutes
3037

3138
/**
32-
* Run Protractor End-to-End Tests for Doc Samples
39+
* Run Protractor End-to-End Tests for a Docs Example
40+
*
41+
* Usage: node run-example-e2e.mjs <examplePath> <yarnPath> <exampleDepsWorkspaceName> [localPackage...]
42+
*
43+
* Args:
44+
* examplePath: path to the example
45+
* yarnPath: path to a vendored version of yarn
46+
* exampleDepsWorkspaceName: name of bazel workspace containing example node_omodules
47+
* localPackages: a vararg of local packages to substitute in place npm deps, in the form @package/name#pathToPackage.
3348
*
3449
* Flags
3550
* --retry to retry failed tests (useful for overcoming flakes)
3651
* e.g. --retry 3 // To try each test up to 3 times.
3752
*/
38-
async function runE2e(examplePath, nodeModulesPath) {
53+
54+
async function runE2e(examplePath) {
3955
const exampleName = path.basename(examplePath);
4056
const maxAttempts = argv.retry || 1;
4157
try {
4258
examplePath = createCopyOfExampleForTest(exampleName, examplePath);
43-
symlinkNodeModules(examplePath, nodeModulesPath);
59+
await constructNodeModules(examplePath);
4460

4561
let testFn;
4662
if (isSystemJsTest(examplePath)) {
@@ -165,8 +181,74 @@ function runProtractorAoT(exampleName, appDir) {
165181
return runProtractorSystemJS(exampleName, promise, appDir, aotRunSpawnInfo);
166182
}
167183

168-
function symlinkNodeModules(examplePath, nodeModulesPath) {
169-
fs.ensureSymlinkSync(nodeModulesPath, path.join(examplePath, 'node_modules'), 'dir');
184+
async function constructNodeModules(examplePath) {
185+
const linkedNodeModules = path.resolve(examplePath, 'node_modules');
186+
const exampleDepsNodeModules = path.resolve('..', EXAMPLE_DEPS_WORKSPACE_NAME, 'node_modules');
187+
fs.ensureDirSync(linkedNodeModules);
188+
189+
await Promise.all([
190+
linkExampleDeps(exampleDepsNodeModules, linkedNodeModules),
191+
linkLocalDeps(exampleDepsNodeModules, linkedNodeModules)
192+
]);
193+
194+
fs.copySync(path.join(exampleDepsNodeModules, '.bin'), path.join(linkedNodeModules, '.bin'));
195+
pointBinSymlinksToLocalPackages(linkedNodeModules);
196+
}
197+
198+
// The .bin folder is copied over from the original yarn_install repository, so the
199+
// bin symlinks point there. When we link local packages in place of their npm equivalent,
200+
// we need to alter those symlinks to point into the local package.
201+
function pointBinSymlinksToLocalPackages(linkedNodeModules) {
202+
if (os.platform() === 'win32') {
203+
// Bins on Windows are not symlinks; they are scripts that will invoke the bin
204+
// relative to their location. The relative path will already point to the symlinked
205+
// local package, so no further action is required.
206+
return;
207+
}
208+
const allNodeModuleBins = globbySync(['**'], {cwd: path.join(linkedNodeModules, '.bin'), onlyFiles: true});
209+
allNodeModuleBins.forEach(bin => {
210+
const symlinkTarget = fs.readlinkSync(path.join(linkedNodeModules, '.bin', bin));
211+
for (const pkgName of Object.keys(LOCAL_PACKAGES)) {
212+
const binMightBeInLocalPackage = symlinkTarget.includes(path.join(EXAMPLE_DEPS_WORKSPACE_NAME, 'node_modules', pkgName) + path.sep);
213+
if (binMightBeInLocalPackage) {
214+
const pathToBinWithinPackage = symlinkTarget.substring(symlinkTarget.indexOf(pkgName) + pkgName.length + path.sep.length);
215+
const binExistsInLocalPackage = fs.existsSync(path.join(linkedNodeModules, pkgName, pathToBinWithinPackage));
216+
if (binExistsInLocalPackage) {
217+
// Replace the copied bin symlink with one that points to the symlinked local package.
218+
fs.rmSync(path.join(linkedNodeModules, '.bin', bin));
219+
fs.ensureSymlinkSync(path.join('..', pkgName, pathToBinWithinPackage), path.join(linkedNodeModules, '.bin', bin));
220+
}
221+
break;
222+
}
223+
}
224+
});
225+
}
226+
227+
function linkExampleDeps(exampleDepsNodeModules, linkedNodeModules) {
228+
const exampleDepsPackages = getPackageNamesFromNodeModules(exampleDepsNodeModules);
229+
230+
return Promise.all(exampleDepsPackages
231+
.filter(pkgName => !(pkgName in LOCAL_PACKAGES))
232+
.map(pkgName => fs.ensureSymlink(path.join(exampleDepsNodeModules, pkgName), path.join(linkedNodeModules, pkgName), 'dir'))
233+
);
234+
}
235+
236+
async function linkLocalDeps(exampleDepsNodeModules, linkedNodeModules) {
237+
const hasNpmDepForPkg = await Promise.all(Object.keys(LOCAL_PACKAGES).map(pkgName => fs.pathExists(path.join(exampleDepsNodeModules, pkgName))));
238+
239+
return Promise.all(Object.keys(LOCAL_PACKAGES).filter((pkgName, i) => hasNpmDepForPkg[i])
240+
.map(pkgName => fs.ensureSymlinkSync(LOCAL_PACKAGES[pkgName], path.join(linkedNodeModules, pkgName), 'dir')));
241+
}
242+
243+
function getPackageNamesFromNodeModules(nodeModulesPath) {
244+
return globbySync([
245+
'@*/*',
246+
'!@*$', // Exclude a namespace folder itself
247+
'(?!@)*',
248+
'!.bin',
249+
'!.yarn-integrity',
250+
'!_*'
251+
], {cwd: nodeModulesPath, onlyDirectories: true, dot: true});
170252
}
171253

172254
// Start the example in appDir; then run protractor with the specified
@@ -188,6 +270,16 @@ function runE2eTestsCLI(exampleName, appDir) {
188270
}
189271
}
190272

273+
// When local packages are symlinked in, node has trouble resolving some peer deps. Setting
274+
// preserveSymlinks: true in angular.json fixes this. This isn't required without local
275+
// packages because in the worst case we would leak into the original Bazel repository
276+
// and it would still find a node_modules folder for resolution.
277+
if (Object.keys(LOCAL_PACKAGES).length > 0) {
278+
const angularJson = jsonc.load(path.join(appDir, 'angular.json'), {encoding: 'utf-8'});
279+
angularJson.projects['angular.io-example'].architect.build.options.preserveSymlinks = true;
280+
fs.writeFileSync(path.join(appDir, 'angular.json'), JSON.stringify(angularJson));
281+
}
282+
191283
// `--no-webdriver-update` is needed to preserve the ChromeDriver version already installed.
192284
const testCommands = config.tests || [{
193285
cmd: NODE,
@@ -306,4 +398,5 @@ function adjustChromeBinPathForWindows() {
306398
return process.env.CHROME_BIN;
307399
}
308400

309-
runE2e(EXAMPLE_PATH, NODE_MODULES_PATH);
401+
runE2e(EXAMPLE_PATH);
402+

‎aio/tools/examples/shared/BUILD.bazel‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,3 @@
1-
load("//aio/tools/ng-packages-installer:node_modules.bzl", "node_modules")
2-
load("//:packages.bzl", "AIO_EXAMPLE_PACKAGES", "to_package_label")
3-
41
package(default_visibility = ["//visibility:public"])
52

63
filegroup(
@@ -10,12 +7,3 @@ filegroup(
107
exclude = ["BUILD.bazel"],
118
),
129
)
13-
14-
node_modules(
15-
name = "node_modules",
16-
)
17-
18-
node_modules(
19-
name = "local/node_modules",
20-
local_package_substitutions = [to_package_label(pkg) for pkg in AIO_EXAMPLE_PACKAGES],
21-
)

‎aio/tools/ng-packages-installer/.eslintrc.js‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎aio/tools/ng-packages-installer/BUILD.bazel‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

0 commit comments

Comments
 (0)