Skip to content

Commit a8ab7e8

Browse files
skhkibanamachine
skh
and
kibanamachine
committed
[Ingest Manager] Install uploaded package (#77986)
* Refactor: installPackage -> installPackageFromRegistry * Refactor: factor out source-agnostic installation steps * Unpack and cache uploaded zip and tgz files. * Add basic archive verification and parse manifest. * Catch error when zip archive is uploaded as gzip. * Add API integration tests. * Remove unnecessary use of "package key" concept. * Add 'install_source' property to saved object epm-packages. * Adjust tests. * Add API integration test for manifest missing fields. * Refactor loadArchive -> loadArchivePackage. * Refactor caching of package archive content * Get datasets and config templates from manifest files. * Use file paths from archive instead of asset paths from registry. * Correctly load registry packages into cache * Use InstallablePackage instead of RegistryPackage where possible. * Actually install uploaded package. * Add missing field to saved objects in tests. * Adjust unit test to pick buffer extractor. * Adjust unit test. * Fix and re-enable getAsset() test. * Adjust integration tests. * Make error message match test. * Pick data_stream.dataset from manifest if set. * dataset -> data_stream also in comments * Remove unused variable. * Use pkgToPkgKey() where appropriate. * More dataset -> data stream renaming. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
1 parent fcc6e0e commit a8ab7e8

35 files changed

Lines changed: 814 additions & 189 deletions

‎x-pack/plugins/ingest_manager/common/types/models/epm.ts‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export enum InstallStatus {
2020
}
2121

2222
export type InstallType = 'reinstall' | 'reupdate' | 'rollback' | 'update' | 'install';
23+
export type InstallSource = 'registry' | 'upload';
2324

2425
export type EpmPackageInstallStatus = 'installed' | 'installing';
2526

@@ -49,10 +50,8 @@ export enum AgentAssetType {
4950

5051
export type RegistryRelease = 'ga' | 'beta' | 'experimental';
5152

52-
// from /package/{name}
53-
// type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go
54-
// https://github.com/elastic/package-registry/blob/master/docs/api/package.json
55-
export interface RegistryPackage {
53+
// Fields common to packages that come from direct upload and the registry
54+
export interface InstallablePackage {
5655
name: string;
5756
title?: string;
5857
version: string;
@@ -61,14 +60,24 @@ export interface RegistryPackage {
6160
description: string;
6261
type: string;
6362
categories: string[];
64-
requirement: RequirementsByServiceName;
6563
screenshots?: RegistryImage[];
6664
icons?: RegistryImage[];
6765
assets?: string[];
6866
internal?: boolean;
6967
format_version: string;
7068
data_streams?: RegistryDataStream[];
7169
policy_templates?: RegistryPolicyTemplate[];
70+
}
71+
72+
// Uploaded package archives don't have extra fields
73+
// Linter complaint disabled because this extra type is meant for better code readability
74+
// eslint-disable-next-line @typescript-eslint/no-empty-interface
75+
export interface ArchivePackage extends InstallablePackage {}
76+
77+
// Registry packages do have extra fields.
78+
// cf. type Package struct at https://github.com/elastic/package-registry/blob/master/util/package.go
79+
export interface RegistryPackage extends InstallablePackage {
80+
requirement: RequirementsByServiceName;
7281
download: string;
7382
path: string;
7483
}
@@ -240,6 +249,7 @@ export interface Installation extends SavedObjectAttributes {
240249
install_status: EpmPackageInstallStatus;
241250
install_version: string;
242251
install_started_at: string;
252+
install_source: InstallSource;
243253
}
244254

245255
export type Installable<T> = Installed<T> | NotInstalled<T>;

‎x-pack/plugins/ingest_manager/server/errors/handlers.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
IngestManagerError,
1414
RegistryError,
1515
PackageNotFoundError,
16+
PackageUnsupportedMediaTypeError,
1617
defaultIngestErrorHandler,
1718
} from './index';
1819

@@ -101,6 +102,25 @@ describe('defaultIngestErrorHandler', () => {
101102
expect(mockContract.logger?.error).toHaveBeenCalledWith(error.message);
102103
});
103104

105+
it('415: PackageUnsupportedMediaType', async () => {
106+
const error = new PackageUnsupportedMediaTypeError('123');
107+
const response = httpServerMock.createResponseFactory();
108+
109+
await defaultIngestErrorHandler({ error, response });
110+
111+
// response
112+
expect(response.ok).toHaveBeenCalledTimes(0);
113+
expect(response.customError).toHaveBeenCalledTimes(1);
114+
expect(response.customError).toHaveBeenCalledWith({
115+
statusCode: 415,
116+
body: { message: error.message },
117+
});
118+
119+
// logging
120+
expect(mockContract.logger?.error).toHaveBeenCalledTimes(1);
121+
expect(mockContract.logger?.error).toHaveBeenCalledWith(error.message);
122+
});
123+
104124
it('404: PackageNotFoundError', async () => {
105125
const error = new PackageNotFoundError('123');
106126
const response = httpServerMock.createResponseFactory();

‎x-pack/plugins/ingest_manager/server/errors/handlers.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@ import {
1313
} from 'src/core/server';
1414
import { errors as LegacyESErrors } from 'elasticsearch';
1515
import { appContextService } from '../services';
16-
import { IngestManagerError, RegistryError, PackageNotFoundError } from './index';
16+
import {
17+
IngestManagerError,
18+
RegistryError,
19+
PackageNotFoundError,
20+
PackageUnsupportedMediaTypeError,
21+
} from './index';
1722

1823
type IngestErrorHandler = (
1924
params: IngestErrorHandlerParams
@@ -52,6 +57,9 @@ const getHTTPResponseCode = (error: IngestManagerError): number => {
5257
if (error instanceof PackageNotFoundError) {
5358
return 404; // Not Found
5459
}
60+
if (error instanceof PackageUnsupportedMediaTypeError) {
61+
return 415; // Unsupported Media Type
62+
}
5563

5664
return 400; // Bad Request
5765
};

‎x-pack/plugins/ingest_manager/server/errors/index.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,7 @@ export class RegistryConnectionError extends RegistryError {}
1818
export class RegistryResponseError extends RegistryError {}
1919
export class PackageNotFoundError extends IngestManagerError {}
2020
export class PackageOutdatedError extends IngestManagerError {}
21+
export class PackageUnsupportedMediaTypeError extends IngestManagerError {}
22+
export class PackageInvalidArchiveError extends IngestManagerError {}
23+
export class PackageCacheError extends IngestManagerError {}
24+
export class PackageOperationNotSupportedError extends IngestManagerError {}

‎x-pack/plugins/ingest_manager/server/routes/epm/handlers.ts‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import { RequestHandler, CustomHttpResponseOptions } from 'src/core/server';
88
import {
99
GetInfoResponse,
1010
InstallPackageResponse,
11-
MessageResponse,
1211
DeletePackageResponse,
1312
GetCategoriesResponse,
1413
GetPackagesResponse,
@@ -35,8 +34,9 @@ import {
3534
getFile,
3635
getPackageInfo,
3736
handleInstallPackageFailure,
38-
installPackage,
3937
isBulkInstallError,
38+
installPackageFromRegistry,
39+
installPackageByUpload,
4040
removeInstallation,
4141
getLimitedPackages,
4242
getInstallationObject,
@@ -148,7 +148,7 @@ export const installPackageFromRegistryHandler: RequestHandler<
148148
const { pkgName, pkgVersion } = splitPkgKey(pkgkey);
149149
const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
150150
try {
151-
const res = await installPackage({
151+
const res = await installPackageFromRegistry({
152152
savedObjectsClient,
153153
pkgkey,
154154
callCluster,
@@ -212,10 +212,24 @@ export const installPackageByUploadHandler: RequestHandler<
212212
undefined,
213213
TypeOf<typeof InstallPackageByUploadRequestSchema.body>
214214
> = async (context, request, response) => {
215-
const body: MessageResponse = {
216-
response: 'package upload was received ok, but not installed (not implemented yet)',
217-
};
218-
return response.ok({ body });
215+
const savedObjectsClient = context.core.savedObjects.client;
216+
const callCluster = context.core.elasticsearch.legacy.client.callAsCurrentUser;
217+
const contentType = request.headers['content-type'] as string; // from types it could also be string[] or undefined but this is checked later
218+
const archiveBuffer = Buffer.from(request.body);
219+
try {
220+
const res = await installPackageByUpload({
221+
savedObjectsClient,
222+
callCluster,
223+
archiveBuffer,
224+
contentType,
225+
});
226+
const body: InstallPackageResponse = {
227+
response: res,
228+
};
229+
return response.ok({ body });
230+
} catch (error) {
231+
return defaultIngestErrorHandler({ error, response });
232+
}
219233
};
220234

221235
export const deletePackageHandler: RequestHandler<TypeOf<

‎x-pack/plugins/ingest_manager/server/saved_objects/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,7 @@ const getSavedObjectTypes = (
303303
install_started_at: { type: 'date' },
304304
install_version: { type: 'keyword' },
305305
install_status: { type: 'keyword' },
306+
install_source: { type: 'keyword' },
306307
},
307308
},
308309
},

‎x-pack/plugins/ingest_manager/server/saved_objects/migrations/to_v7_10_0.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
EnrollmentAPIKey,
1515
Settings,
1616
AgentAction,
17+
Installation,
1718
} from '../../types';
1819

1920
export const migrateAgentToV7100: SavedObjectMigrationFn<
@@ -134,3 +135,12 @@ export const migrateAgentActionToV7100 = (
134135
}
135136
);
136137
};
138+
139+
export const migrateInstallationToV7100: SavedObjectMigrationFn<
140+
Exclude<Installation, 'install_source'>,
141+
Installation
142+
> = (installationDoc) => {
143+
installationDoc.attributes.install_source = 'registry';
144+
145+
return installationDoc;
146+
};

0 commit comments

Comments
 (0)