Skip to content

Commit 245cc09

Browse files
Spencerspalger
Spencer
authored and
spalger
committed
[kbn/pm] add caching to bootstrap (#53622)
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> # Conflicts: # packages/kbn-pm/dist/index.js
1 parent 0ae2705 commit 245cc09

29 files changed

Lines changed: 68989 additions & 34177 deletions

‎packages/kbn-dev-utils/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"execa": "^3.2.0",
1616
"exit-hook": "^2.2.0",
1717
"getopts": "^2.2.5",
18+
"load-json-file": "^6.2.0",
1819
"moment": "^2.24.0",
1920
"rxjs": "^6.5.3",
2021
"tree-kill": "^1.2.1",

‎packages/kbn-dev-utils/src/constants.ts‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎packages/kbn-dev-utils/src/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,6 @@ export {
2727
export { createAbsolutePathSerializer } from './serializers';
2828
export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';
2929
export { run, createFailError, createFlagError, combineErrors, isFailError, Flags } from './run';
30-
export { REPO_ROOT } from './constants';
30+
export { REPO_ROOT } from './repo_root';
3131
export { KbnClient } from './kbn_client';
3232
export * from './axios';

‎packages/kbn-dev-utils/src/repo_root.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import Path from 'path';
21+
import Fs from 'fs';
22+
23+
import loadJsonFile from 'load-json-file';
24+
25+
const isKibanaDir = (dir: string) => {
26+
try {
27+
const path = Path.resolve(dir, 'package.json');
28+
const json = loadJsonFile.sync(path);
29+
if (json && typeof json === 'object' && 'name' in json && json.name === 'kibana') {
30+
return true;
31+
}
32+
} catch (error) {
33+
if (error && error.code === 'ENOENT') {
34+
return false;
35+
}
36+
37+
throw error;
38+
}
39+
};
40+
41+
// search for the kibana directory, since this file is moved around it might
42+
// not be where we think but should always be a relatively close parent
43+
// of this directory
44+
const startDir = Fs.realpathSync(__dirname);
45+
const { root: rootDir } = Path.parse(startDir);
46+
let cursor = startDir;
47+
while (true) {
48+
if (isKibanaDir(cursor)) {
49+
break;
50+
}
51+
52+
const parent = Path.dirname(cursor);
53+
if (parent === rootDir) {
54+
throw new Error(`unable to find kibana directory from ${startDir}`);
55+
}
56+
cursor = parent;
57+
}
58+
59+
export const REPO_ROOT = cursor;

‎packages/kbn-pm/dist/index.js‎

Lines changed: 68267 additions & 34109 deletions
Large diffs are not rendered by default.

‎packages/kbn-pm/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
"@types/tempy": "^0.2.0",
3535
"@types/wrap-ansi": "^2.0.15",
3636
"@types/write-pkg": "^3.1.0",
37+
"@kbn/dev-utils": "1.0.0",
38+
"@yarnpkg/lockfile": "^1.1.0",
3739
"babel-loader": "^8.0.6",
3840
"chalk": "^2.4.2",
3941
"cmd-shim": "^2.1.0",
@@ -48,6 +50,7 @@
4850
"indent-string": "^3.2.0",
4951
"lodash.clonedeepwith": "^4.5.0",
5052
"log-symbols": "^2.2.0",
53+
"multimatch": "^4.0.0",
5154
"ncp": "^2.0.0",
5255
"ora": "^1.4.0",
5356
"prettier": "^1.19.1",

‎packages/kbn-pm/src/cli.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ function help() {
4747
-i, --include Include only specified projects. If left unspecified, it defaults to including all projects.
4848
--oss Do not include the x-pack when running command.
4949
--skip-kibana-plugins Filter all plugins in ./plugins and ../kibana-extra when running command.
50+
--no-cache Disable the bootstrap cache
5051
`);
5152
}
5253

@@ -65,7 +66,10 @@ export async function run(argv: string[]) {
6566
h: 'help',
6667
i: 'include',
6768
},
68-
boolean: ['prefer-offline', 'frozen-lockfile'],
69+
default: {
70+
cache: true,
71+
},
72+
boolean: ['prefer-offline', 'frozen-lockfile', 'cache'],
6973
});
7074

7175
const args = options._;

‎packages/kbn-pm/src/commands/__snapshots__/bootstrap.test.ts.snap‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/kbn-pm/src/commands/bootstrap.test.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import { Project } from '../utils/project';
2929
import { buildProjectGraph } from '../utils/projects';
3030
import { installInDir, runScriptInPackageStreaming, yarnWorkspacesInfo } from '../utils/scripts';
3131
import { BootstrapCommand } from './bootstrap';
32+
import { Kibana } from '../utils/kibana';
3233

3334
const mockInstallInDir = installInDir as jest.Mock;
3435
const mockRunScriptInPackageStreaming = runScriptInPackageStreaming as jest.Mock;
@@ -107,6 +108,7 @@ test('handles dependencies of dependencies', async () => {
107108
['bar', bar],
108109
['baz', baz],
109110
]);
111+
const kbn = new Kibana(projects);
110112
const projectGraph = buildProjectGraph(projects);
111113

112114
const logMock = jest.spyOn(console, 'log').mockImplementation(noop);
@@ -115,6 +117,7 @@ test('handles dependencies of dependencies', async () => {
115117
extraArgs: [],
116118
options: {},
117119
rootPath: '',
120+
kbn,
118121
});
119122

120123
expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -142,6 +145,7 @@ test('does not run installer if no deps in package', async () => {
142145
['kibana', kibana],
143146
['bar', bar],
144147
]);
148+
const kbn = new Kibana(projects);
145149
const projectGraph = buildProjectGraph(projects);
146150

147151
const logMock = jest.spyOn(console, 'log').mockImplementation(noop);
@@ -150,6 +154,7 @@ test('does not run installer if no deps in package', async () => {
150154
extraArgs: [],
151155
options: {},
152156
rootPath: '',
157+
kbn,
153158
});
154159

155160
expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -167,6 +172,7 @@ test('handles "frozen-lockfile"', async () => {
167172
});
168173

169174
const projects = new Map([['kibana', kibana]]);
175+
const kbn = new Kibana(projects);
170176
const projectGraph = buildProjectGraph(projects);
171177

172178
jest.spyOn(console, 'log').mockImplementation(noop);
@@ -177,6 +183,7 @@ test('handles "frozen-lockfile"', async () => {
177183
'frozen-lockfile': true,
178184
},
179185
rootPath: '',
186+
kbn,
180187
});
181188

182189
expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -205,6 +212,7 @@ test('calls "kbn:bootstrap" scripts and links executables after installing deps'
205212
['kibana', kibana],
206213
['bar', bar],
207214
]);
215+
const kbn = new Kibana(projects);
208216
const projectGraph = buildProjectGraph(projects);
209217

210218
jest.spyOn(console, 'log').mockImplementation(noop);
@@ -213,6 +221,7 @@ test('calls "kbn:bootstrap" scripts and links executables after installing deps'
213221
extraArgs: [],
214222
options: {},
215223
rootPath: '',
224+
kbn,
216225
});
217226

218227
expect(mockLinkProjectExecutables.mock.calls).toMatchSnapshot('link bins');

‎packages/kbn-pm/src/commands/bootstrap.ts‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,14 @@ import { log } from '../utils/log';
2424
import { parallelizeBatches } from '../utils/parallelize';
2525
import { topologicallyBatchProjects } from '../utils/projects';
2626
import { ICommand } from './';
27+
import { getAllChecksums } from '../utils/project_checksums';
28+
import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
2729

2830
export const BootstrapCommand: ICommand = {
2931
description: 'Install dependencies and crosslink projects',
3032
name: 'bootstrap',
3133

32-
async run(projects, projectGraph, { options }) {
34+
async run(projects, projectGraph, { options, kbn }) {
3335
const batchedProjectsByWorkspace = topologicallyBatchProjects(projects, projectGraph, {
3436
batchByWorkspace: true,
3537
});
@@ -65,9 +67,18 @@ export const BootstrapCommand: ICommand = {
6567
* have to, as it will slow down the bootstrapping process.
6668
*/
6769
log.write(chalk.bold('\nLinking executables completed, running `kbn:bootstrap` scripts\n'));
68-
await parallelizeBatches(batchedProjects, async pkg => {
69-
if (pkg.hasScript('kbn:bootstrap')) {
70-
await pkg.runScriptStreaming('kbn:bootstrap');
70+
71+
const checksums = options.cache ? await getAllChecksums(kbn, log) : false;
72+
await parallelizeBatches(batchedProjects, async project => {
73+
if (project.hasScript('kbn:bootstrap')) {
74+
const cacheFile = new BootstrapCacheFile(kbn, project, checksums);
75+
if (cacheFile.isValid()) {
76+
log.success(`[${project.name}] cache up to date`);
77+
} else {
78+
cacheFile.delete();
79+
await project.runScriptStreaming('kbn:bootstrap');
80+
cacheFile.write();
81+
}
7182
}
7283
});
7384

0 commit comments

Comments
 (0)