Skip to content

Commit d9c7c8b

Browse files
spalgerspalger
committed
[kbn/optimizer] handle lower-power systems better
1 parent cffdc72 commit d9c7c8b

15 files changed

Lines changed: 388 additions & 159 deletions

‎packages/kbn-optimizer/README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@ The `@kbn/optimizer` is automatically executed from the dev cli, the Kibana buil
2020

2121
### Worker count
2222

23-
You can limit the number of workers the optimizer uses by setting the `KBN_OPTIMIZER_MAX_WORKERS` environment variable. You might want to do this if your system struggles to keep up while the optimizer is getting started and building all plugins as fast as possible. Setting `KBN_OPTIMIZER_MAX_WORKERS=1` will cause the optimizer to take the longest amount of time but will have the smallest impact on other components of your system.
23+
You can limit the number of workers the optimizer uses by setting the `KBN_OPTIMIZER_MAX_ACTIVE_WORKERS` environment variable. You might want to do this if your system struggles to keep up while the optimizer is getting started and building all plugins as fast as possible. Setting `KBN_OPTIMIZER_MAX_ACTIVE_WORKERS=1` will cause the optimizer to take the longest amount of time but will have the smallest impact on other components of your system.
2424

2525
We only limit the number of workers we will start at any given time. If we start more workers later we will limit the number of workers we start at that time by the maximum, but we don't take into account the number of workers already started because it is assumed that those workers are doing very little work. This greatly simplifies the logic as we don't ever have to reallocate workers and provides the best performance in most cases.
2626

27+
You can also influce the number of works using the `KBN_OPTIMIZER_TARGET_MODULES_PER_WORKER` environment variable. This value defaults to 3000 and will cause the creation of enough workers so that each worker will be building approximately this many bundles. If the number of workers created by this value is greater than the maximum allowed by `KBN_OPTIMIZER_MAX_ACTIVE_WORKERS` then we will only start the maximum allowed active workers initially and start subsequent workers once the first ones are done (or idle when in `watch` mode).
28+
2729
### Caching
2830

2931
Bundles built by the the optimizer include a cache file which describes the information needed to determine if the bundle needs to be rebuilt when the optimizer is restarted. Caching is enabled by default and is very aggressive about invalidating the cache output, but if you need to disable caching you can pass `--no-cache` to `node scripts/build_kibana_platform_plugins`, or set the `KBN_OPTIMIZER_NO_CACHE` environment variable to anything (env overrides everything).

‎packages/kbn-optimizer/src/cli.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,11 @@ run(
7070
throw createFlagError('expected --no-inspect-workers to have no value');
7171
}
7272

73-
const maxWorkerCount = flags.workers ? Number.parseInt(String(flags.workers), 10) : undefined;
74-
if (maxWorkerCount !== undefined && (!Number.isFinite(maxWorkerCount) || maxWorkerCount < 1)) {
73+
const maxActiveWorkers = flags.workers ? Number.parseInt(String(flags.workers), 10) : undefined;
74+
if (
75+
maxActiveWorkers !== undefined &&
76+
(!Number.isFinite(maxActiveWorkers) || maxActiveWorkers < 1)
77+
) {
7578
throw createFlagError('expected --workers to be a number greater than 0');
7679
}
7780

@@ -90,7 +93,7 @@ run(
9093
const config = OptimizerConfig.create({
9194
repoRoot: REPO_ROOT,
9295
watch,
93-
maxWorkerCount,
96+
maxActiveWorkers,
9497
oss,
9598
dist,
9699
cache,

‎packages/kbn-optimizer/src/common/worker_config.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ export interface WorkerConfig {
3131
readonly optimizerCacheKey: unknown;
3232
}
3333

34-
export type CacheableWorkerConfig = Omit<WorkerConfig, 'watch' | 'profileWebpack'>;
34+
export type CacheableWorkerConfig = Omit<WorkerConfig, 'watch' | 'profileWebpack' | 'cache'>;
3535

3636
export function parseWorkerConfig(json: string): WorkerConfig {
3737
try {

‎packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap‎

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
6464
const config = OptimizerConfig.create({
6565
repoRoot: MOCK_REPO_DIR,
6666
pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')],
67-
maxWorkerCount: 1,
67+
maxActiveWorkers: 1,
6868
dist: false,
6969
});
7070

@@ -96,24 +96,30 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
9696
assert('produce two bundle cache events while initializing', bundleCacheStates.length === 2);
9797

9898
const initializedStates = msgs.filter(msg => msg.state.phase === 'initialized');
99-
assert('produce at least one initialized event', initializedStates.length >= 1);
99+
assert(
100+
'produce at least one initialized event',
101+
initializedStates.length >= 1,
102+
initializedStates
103+
);
100104

101105
const workerStarted = msgs.filter(msg => msg.event?.type === 'worker started');
102-
assert('produce one worker started event', workerStarted.length === 1);
106+
assert('produce two worker started events', workerStarted.length === 2, workerStarted);
103107

104108
const runningStates = msgs.filter(msg => msg.state.phase === 'running');
105-
assert(
106-
'produce two or three "running" states',
107-
runningStates.length === 2 || runningStates.length === 3
108-
);
109+
assert('produce four "running" states', runningStates.length === 4, runningStates);
109110

110111
const bundleNotCachedEvents = msgs.filter(msg => msg.event?.type === 'bundle not cached');
111-
assert('produce two "bundle not cached" events', bundleNotCachedEvents.length === 2);
112+
assert(
113+
'produce two "bundle not cached" events',
114+
bundleNotCachedEvents.length === 2,
115+
bundleNotCachedEvents
116+
);
112117

113118
const successStates = msgs.filter(msg => msg.state.phase === 'success');
114119
assert(
115120
'produce one or two "compiler success" states',
116-
successStates.length === 1 || successStates.length === 2
121+
successStates.length === 1 || successStates.length === 2,
122+
successStates
117123
);
118124

119125
const otherStates = msgs.filter(
@@ -167,7 +173,7 @@ it('uses cache on second run and exist cleanly', async () => {
167173
const config = OptimizerConfig.create({
168174
repoRoot: MOCK_REPO_DIR,
169175
pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')],
170-
maxWorkerCount: 1,
176+
maxActiveWorkers: 1,
171177
dist: false,
172178
});
173179

@@ -198,7 +204,7 @@ it('prepares assets for distribution', async () => {
198204
const config = OptimizerConfig.create({
199205
repoRoot: MOCK_REPO_DIR,
200206
pluginScanDirs: [Path.resolve(MOCK_REPO_DIR, 'plugins')],
201-
maxWorkerCount: 1,
207+
maxActiveWorkers: 1,
202208
dist: true,
203209
});
204210

‎packages/kbn-optimizer/src/integration_tests/bundle_cache.test.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ it('emits "bundle cached" event when everything is updated', async () => {
5757
repoRoot: MOCK_REPO_DIR,
5858
pluginScanDirs: [],
5959
pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
60-
maxWorkerCount: 1,
60+
maxActiveWorkers: 1,
6161
});
6262
const [bundle] = config.bundles;
6363

@@ -96,7 +96,7 @@ it('emits "bundle not cached" event when cacheKey is up to date but caching is d
9696
repoRoot: MOCK_REPO_DIR,
9797
pluginScanDirs: [],
9898
pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
99-
maxWorkerCount: 1,
99+
maxActiveWorkers: 1,
100100
cache: false,
101101
});
102102
const [bundle] = config.bundles;
@@ -137,7 +137,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is missing', async ()
137137
repoRoot: MOCK_REPO_DIR,
138138
pluginScanDirs: [],
139139
pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
140-
maxWorkerCount: 1,
140+
maxActiveWorkers: 1,
141141
});
142142
const [bundle] = config.bundles;
143143

@@ -177,7 +177,7 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
177177
repoRoot: MOCK_REPO_DIR,
178178
pluginScanDirs: [],
179179
pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
180-
maxWorkerCount: 1,
180+
maxActiveWorkers: 1,
181181
});
182182
const [bundle] = config.bundles;
183183

@@ -208,8 +208,8 @@ it('emits "bundle not cached" event when optimizerCacheKey is outdated, includes
208208
"diff": "- Expected
209209
+ Received
210210
211-
[32m- old[39m
212-
[31m+ optimizerCacheKey[39m",
211+
[32m- \\"old\\"[39m
212+
[31m+ \\"optimizerCacheKey\\"[39m",
213213
"reason": "optimizer cache key mismatch",
214214
"type": "bundle not cached",
215215
},
@@ -222,7 +222,7 @@ it('emits "bundle not cached" event when cacheKey is missing', async () => {
222222
repoRoot: MOCK_REPO_DIR,
223223
pluginScanDirs: [],
224224
pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
225-
maxWorkerCount: 1,
225+
maxActiveWorkers: 1,
226226
});
227227
const [bundle] = config.bundles;
228228

@@ -260,7 +260,7 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
260260
repoRoot: MOCK_REPO_DIR,
261261
pluginScanDirs: [],
262262
pluginPaths: [Path.resolve(MOCK_REPO_DIR, 'plugins/foo')],
263-
maxWorkerCount: 1,
263+
maxActiveWorkers: 1,
264264
});
265265
const [bundle] = config.bundles;
266266

@@ -291,8 +291,8 @@ it('emits "bundle not cached" event when cacheKey is outdated', async () => {
291291
"diff": "- Expected
292292
+ Received
293293
294-
[32m- old[39m
295-
[31m+ new[39m",
294+
[32m- \\"old\\"[39m
295+
[31m+ \\"new\\"[39m",
296296
"reason": "cache key mismatch",
297297
"type": "bundle not cached",
298298
},

0 commit comments

Comments
 (0)