Skip to content

Commit 4ac9eff

Browse files
author
John Schulz
authored
[Fleet][EPM] Unified install and archive (#83384) (#83944)
## Summary * Further reduce differences between installing uploaded vs registry package * Improve cache/store names, TS types, etc. Including key by name + version + source * Add a cache/store for PackageInfo (e.g. results metadata from registry's /package/version/ response) * Remove ensureCachedArchiveInfo
1 parent 79d4646 commit 4ac9eff

10 files changed

Lines changed: 165 additions & 111 deletions

File tree

‎x-pack/plugins/fleet/server/services/epm/archive/cache.ts‎

Lines changed: 50 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,57 @@
33
* or more contributor license agreements. Licensed under the Elastic License;
44
* you may not use this file except in compliance with the Elastic License.
55
*/
6-
import { pkgToPkgKey } from '../registry/index';
6+
import { ArchiveEntry } from './index';
7+
import { InstallSource, ArchivePackage, RegistryPackage } from '../../../../common';
78

8-
const cache: Map<string, Buffer> = new Map();
9-
export const cacheGet = (key: string) => cache.get(key);
10-
export const cacheSet = (key: string, value: Buffer) => cache.set(key, value);
11-
export const cacheHas = (key: string) => cache.has(key);
12-
export const cacheClear = () => cache.clear();
13-
export const cacheDelete = (key: string) => cache.delete(key);
9+
const archiveEntryCache: Map<ArchiveEntry['path'], ArchiveEntry['buffer']> = new Map();
10+
export const getArchiveEntry = (key: string) => archiveEntryCache.get(key);
11+
export const setArchiveEntry = (key: string, value: Buffer) => archiveEntryCache.set(key, value);
12+
export const hasArchiveEntry = (key: string) => archiveEntryCache.has(key);
13+
export const clearArchiveEntries = () => archiveEntryCache.clear();
14+
export const deleteArchiveEntry = (key: string) => archiveEntryCache.delete(key);
1415

15-
const archiveFilelistCache: Map<string, string[]> = new Map();
16-
export const getArchiveFilelist = (name: string, version: string) =>
17-
archiveFilelistCache.get(pkgToPkgKey({ name, version }));
16+
export interface SharedKey {
17+
name: string;
18+
version: string;
19+
installSource: InstallSource;
20+
}
21+
type SharedKeyString = string;
1822

19-
export const setArchiveFilelist = (name: string, version: string, paths: string[]) =>
20-
archiveFilelistCache.set(pkgToPkgKey({ name, version }), paths);
23+
type ArchiveFilelist = string[];
24+
const archiveFilelistCache: Map<SharedKeyString, ArchiveFilelist> = new Map();
25+
export const getArchiveFilelist = (keyArgs: SharedKey) =>
26+
archiveFilelistCache.get(sharedKey(keyArgs));
2127

22-
export const deleteArchiveFilelist = (name: string, version: string) =>
23-
archiveFilelistCache.delete(pkgToPkgKey({ name, version }));
28+
export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) =>
29+
archiveFilelistCache.set(sharedKey(keyArgs), paths);
30+
31+
export const deleteArchiveFilelist = (keyArgs: SharedKey) =>
32+
archiveFilelistCache.delete(sharedKey(keyArgs));
33+
34+
const packageInfoCache: Map<SharedKeyString, ArchivePackage | RegistryPackage> = new Map();
35+
const sharedKey = ({ name, version, installSource }: SharedKey) =>
36+
`${name}-${version}-${installSource}`;
37+
38+
export const getPackageInfo = (args: SharedKey) => {
39+
const packageInfo = packageInfoCache.get(sharedKey(args));
40+
if (args.installSource === 'registry') {
41+
return packageInfo as RegistryPackage;
42+
} else if (args.installSource === 'upload') {
43+
return packageInfo as ArchivePackage;
44+
} else {
45+
throw new Error(`Unknown installSource: ${args.installSource}`);
46+
}
47+
};
48+
49+
export const setPackageInfo = ({
50+
name,
51+
version,
52+
installSource,
53+
packageInfo,
54+
}: SharedKey & { packageInfo: ArchivePackage | RegistryPackage }) => {
55+
const key = sharedKey({ name, version, installSource });
56+
return packageInfoCache.set(key, packageInfo);
57+
};
58+
59+
export const deletePackageInfo = (args: SharedKey) => packageInfoCache.delete(sharedKey(args));

‎x-pack/plugins/fleet/server/services/epm/archive/index.ts‎

Lines changed: 30 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,68 +4,57 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import { ArchivePackage, AssetParts } from '../../../../common/types';
7+
import { AssetParts, InstallSource } from '../../../../common/types';
88
import { PackageInvalidArchiveError, PackageUnsupportedMediaTypeError } from '../../../errors';
99
import {
10-
cacheGet,
11-
cacheSet,
12-
cacheDelete,
10+
SharedKey,
11+
getArchiveEntry,
12+
setArchiveEntry,
13+
deleteArchiveEntry,
1314
getArchiveFilelist,
1415
setArchiveFilelist,
1516
deleteArchiveFilelist,
17+
deletePackageInfo,
1618
} from './cache';
1719
import { getBufferExtractor } from './extract';
18-
import { parseAndVerifyArchiveEntries } from './validation';
1920

2021
export * from './cache';
21-
export { untarBuffer, unzipBuffer, getBufferExtractor } from './extract';
22+
export { getBufferExtractor, untarBuffer, unzipBuffer } from './extract';
23+
export { parseAndVerifyArchiveBuffer as parseAndVerifyArchiveEntries } from './validation';
2224

2325
export interface ArchiveEntry {
2426
path: string;
2527
buffer?: Buffer;
2628
}
2729

28-
export async function getArchivePackage({
29-
archiveBuffer,
30+
export async function unpackBufferToCache({
31+
name,
32+
version,
3033
contentType,
34+
archiveBuffer,
35+
installSource,
3136
}: {
32-
archiveBuffer: Buffer;
37+
name: string;
38+
version: string;
3339
contentType: string;
34-
}): Promise<{ paths: string[]; archivePackageInfo: ArchivePackage }> {
35-
const entries = await unpackArchiveEntries(archiveBuffer, contentType);
36-
const { archivePackageInfo } = await parseAndVerifyArchiveEntries(entries);
37-
const paths = addEntriesToMemoryStore(entries);
38-
39-
setArchiveFilelist(archivePackageInfo.name, archivePackageInfo.version, paths);
40-
41-
return {
42-
paths,
43-
archivePackageInfo,
44-
};
45-
}
46-
47-
export async function unpackArchiveToCache(
48-
archiveBuffer: Buffer,
49-
contentType: string
50-
): Promise<string[]> {
51-
const entries = await unpackArchiveEntries(archiveBuffer, contentType);
52-
return addEntriesToMemoryStore(entries);
53-
}
54-
55-
function addEntriesToMemoryStore(entries: ArchiveEntry[]) {
40+
archiveBuffer: Buffer;
41+
installSource: InstallSource;
42+
}): Promise<string[]> {
43+
const entries = await unpackBufferEntries(archiveBuffer, contentType);
5644
const paths: string[] = [];
5745
entries.forEach((entry) => {
5846
const { path, buffer } = entry;
5947
if (buffer) {
60-
cacheSet(path, buffer);
48+
setArchiveEntry(path, buffer);
6149
paths.push(path);
6250
}
6351
});
52+
setArchiveFilelist({ name, version, installSource }, paths);
6453

6554
return paths;
6655
}
6756

68-
export async function unpackArchiveEntries(
57+
export async function unpackBufferEntries(
6958
archiveBuffer: Buffer,
7059
contentType: string
7160
): Promise<ArchiveEntry[]> {
@@ -96,16 +85,18 @@ export async function unpackArchiveEntries(
9685
return entries;
9786
}
9887

99-
export const deletePackageCache = (name: string, version: string) => {
88+
export const deletePackageCache = ({ name, version, installSource }: SharedKey) => {
10089
// get cached archive filelist
101-
const paths = getArchiveFilelist(name, version);
90+
const paths = getArchiveFilelist({ name, version, installSource });
10291

10392
// delete cached archive filelist
104-
deleteArchiveFilelist(name, version);
93+
deleteArchiveFilelist({ name, version, installSource });
10594

10695
// delete cached archive files
107-
// this has been populated in unpackArchiveToCache()
108-
paths?.forEach((path) => cacheDelete(path));
96+
// this has been populated in unpackBufferToCache()
97+
paths?.forEach(deleteArchiveEntry);
98+
99+
deletePackageInfo({ name, version, installSource });
109100
};
110101

111102
export function getPathParts(path: string): AssetParts {
@@ -139,7 +130,7 @@ export function getPathParts(path: string): AssetParts {
139130
}
140131

141132
export function getAsset(key: string) {
142-
const buffer = cacheGet(key);
133+
const buffer = getArchiveEntry(key);
143134
if (buffer === undefined) throw new Error(`Cannot find asset ${key}`);
144135

145136
return buffer;

‎x-pack/plugins/fleet/server/services/epm/archive/validation.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {
1515
RegistryVarsEntry,
1616
} from '../../../../common/types';
1717
import { PackageInvalidArchiveError } from '../../../errors';
18-
import { ArchiveEntry } from './index';
18+
import { unpackBufferEntries } from './index';
1919
import { pkgToPkgKey } from '../registry';
2020

2121
const MANIFESTS: Record<string, Buffer> = {};
@@ -24,22 +24,24 @@ const MANIFEST_NAME = 'manifest.yml';
2424
// TODO: everything below performs verification of manifest.yml files, and hence duplicates functionality already implemented in the
2525
// package registry. At some point this should probably be replaced (or enhanced) with verification based on
2626
// https://github.com/elastic/package-spec/
27-
export async function parseAndVerifyArchiveEntries(
28-
entries: ArchiveEntry[]
29-
): Promise<{ paths: string[]; archivePackageInfo: ArchivePackage }> {
27+
export async function parseAndVerifyArchiveBuffer(
28+
archiveBuffer: Buffer,
29+
contentType: string
30+
): Promise<{ paths: string[]; packageInfo: ArchivePackage }> {
31+
const entries = await unpackBufferEntries(archiveBuffer, contentType);
3032
const paths: string[] = [];
3133
entries.forEach(({ path, buffer }) => {
3234
paths.push(path);
3335
if (path.endsWith(MANIFEST_NAME) && buffer) MANIFESTS[path] = buffer;
3436
});
3537

3638
return {
37-
archivePackageInfo: parseAndVerifyArchive(paths),
39+
packageInfo: parseAndVerifyArchive(paths),
3840
paths,
3941
};
4042
}
4143

42-
export function parseAndVerifyArchive(paths: string[]): ArchivePackage {
44+
function parseAndVerifyArchive(paths: string[]): ArchivePackage {
4345
// The top-level directory must match pkgName-pkgVersion, and no other top-level files or directories may be present
4446
const toplevelDir = paths[0].split('/')[0];
4547
paths.forEach((path) => {

‎x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,14 +85,6 @@ export async function installIndexPatterns(
8585
savedObjectsClient,
8686
installationStatuses.Installed
8787
);
88-
// TODO: move to install package
89-
// cache all installed packages if they don't exist
90-
const packagePromises = installedPackages.map((pkg) =>
91-
// TODO: this hard-codes 'registry' as installSource, so uploaded packages are ignored
92-
// and their fields will be removed from the generated index patterns after this runs.
93-
Registry.ensureCachedArchiveInfo(pkg.pkgName, pkg.pkgVersion, 'registry')
94-
);
95-
await Promise.all(packagePromises);
9688

9789
const packageVersionsToFetch = [...installedPackages];
9890
if (pkgName && pkgVersion) {

‎x-pack/plugins/fleet/server/services/epm/packages/assets.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
*/
66

77
import { InstallablePackage } from '../../../types';
8-
import * as Registry from '../registry';
98
import { ArchiveEntry, getArchiveFilelist, getAsset } from '../archive';
109

1110
// paths from RegistryPackage are routes to the assets on EPR
@@ -21,7 +20,8 @@ export function getAssets(
2120
datasetName?: string
2221
): string[] {
2322
const assets: string[] = [];
24-
const paths = getArchiveFilelist(packageInfo.name, packageInfo.version);
23+
const { name, version } = packageInfo;
24+
const paths = getArchiveFilelist({ name, version, installSource: 'registry' });
2525
// TODO: might be better to throw a PackageCacheError here
2626
if (!paths || paths.length === 0) return assets;
2727

@@ -47,15 +47,13 @@ export function getAssets(
4747
return assets;
4848
}
4949

50+
// ASK: Does getAssetsData need an installSource now?
51+
// if so, should it be an Installation vs InstallablePackage or add another argument?
5052
export async function getAssetsData(
5153
packageInfo: InstallablePackage,
5254
filter = (path: string): boolean => true,
5355
datasetName?: string
5456
): Promise<ArchiveEntry[]> {
55-
// TODO: Needs to be called to fill the cache but should not be required
56-
57-
await Registry.ensureCachedArchiveInfo(packageInfo.name, packageInfo.version, 'registry');
58-
5957
// Gather all asset data
6058
const assets = getAssets(packageInfo, filter, datasetName);
6159
const entries: ArchiveEntry[] = assets.map((path) => {

‎x-pack/plugins/fleet/server/services/epm/packages/get.ts‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,11 +109,7 @@ export async function getPackageInfo(options: {
109109
pkgVersion: string;
110110
}): Promise<PackageInfo> {
111111
const { savedObjectsClient, pkgName, pkgVersion } = options;
112-
const [
113-
savedObject,
114-
latestPackage,
115-
{ paths: assets, registryPackageInfo: item },
116-
] = await Promise.all([
112+
const [savedObject, latestPackage, { paths: assets, packageInfo: item }] = await Promise.all([
117113
getInstallationObject({ savedObjectsClient, pkgName }),
118114
Registry.fetchFindLatestPackage(pkgName),
119115
Registry.getRegistryPackage(pkgName, pkgVersion),

‎x-pack/plugins/fleet/server/services/epm/packages/install.ts‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import {
2828
KibanaAssetType,
2929
} from '../../../types';
3030
import * as Registry from '../registry';
31+
import { setPackageInfo, parseAndVerifyArchiveEntries, unpackBufferToCache } from '../archive';
3132
import {
3233
getInstallation,
3334
getInstallationObject,
@@ -43,7 +44,6 @@ import {
4344
} from '../../../errors';
4445
import { getPackageSavedObjects } from './get';
4546
import { appContextService } from '../../app_context';
46-
import { getArchivePackage } from '../archive';
4747
import { _installPackage } from './_install_package';
4848

4949
export async function installLatestPackage(options: {
@@ -245,29 +245,26 @@ async function installPackageFromRegistry({
245245
}: InstallRegistryPackageParams): Promise<AssetReference[]> {
246246
// TODO: change epm API to /packageName/version so we don't need to do this
247247
const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey);
248-
// TODO: calls to getInstallationObject, Registry.fetchInfo, and Registry.fetchFindLatestPackge
249-
// and be replaced by getPackageInfo after adjusting for it to not group/use archive assets
250-
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
251248
// get the currently installed package
252249
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
253-
254250
const installType = getInstallType({ pkgVersion, installedPkg });
255-
256251
// let the user install if using the force flag or needing to reinstall or install a previous version due to failed update
257252
const installOutOfDateVersionOk =
258253
installType === 'reinstall' || installType === 'reupdate' || installType === 'rollback';
254+
255+
const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
259256
if (semverLt(pkgVersion, latestPackage.version) && !force && !installOutOfDateVersionOk) {
260257
throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
261258
}
262259

263-
const { paths, registryPackageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
260+
const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
264261

265262
return _installPackage({
266263
savedObjectsClient,
267264
callCluster,
268265
installedPkg,
269266
paths,
270-
packageInfo: registryPackageInfo,
267+
packageInfo,
271268
installType,
272269
installSource: 'registry',
273270
});
@@ -290,27 +287,44 @@ async function installPackageByUpload({
290287
archiveBuffer,
291288
contentType,
292289
}: InstallUploadedArchiveParams): Promise<AssetReference[]> {
293-
const { paths, archivePackageInfo } = await getArchivePackage({ archiveBuffer, contentType });
290+
const { packageInfo } = await parseAndVerifyArchiveEntries(archiveBuffer, contentType);
294291

295292
const installedPkg = await getInstallationObject({
296293
savedObjectsClient,
297-
pkgName: archivePackageInfo.name,
294+
pkgName: packageInfo.name,
298295
});
299-
const installType = getInstallType({ pkgVersion: archivePackageInfo.version, installedPkg });
296+
297+
const installType = getInstallType({ pkgVersion: packageInfo.version, installedPkg });
300298
if (installType !== 'install') {
301299
throw new PackageOperationNotSupportedError(
302-
`Package upload only supports fresh installations. Package ${archivePackageInfo.name} is already installed, please uninstall first.`
300+
`Package upload only supports fresh installations. Package ${packageInfo.name} is already installed, please uninstall first.`
303301
);
304302
}
305303

304+
const installSource = 'upload';
305+
const paths = await unpackBufferToCache({
306+
name: packageInfo.name,
307+
version: packageInfo.version,
308+
installSource,
309+
archiveBuffer,
310+
contentType,
311+
});
312+
313+
setPackageInfo({
314+
name: packageInfo.name,
315+
version: packageInfo.version,
316+
installSource,
317+
packageInfo,
318+
});
319+
306320
return _installPackage({
307321
savedObjectsClient,
308322
callCluster,
309323
installedPkg,
310324
paths,
311-
packageInfo: archivePackageInfo,
325+
packageInfo,
312326
installType,
313-
installSource: 'upload',
327+
installSource,
314328
});
315329
}
316330

0 commit comments

Comments
 (0)