Skip to content

Commit 1cd477a

Browse files
Corey RobertsonelasticmachinekibanamachineJohn Schulz
authored
[Fleet] Allow snake cased Kibana assets (#77515)
* Properly handle kibana assets with underscores in their path * Recomment test * Fix type check * Don't install index patterns that are reserved * Introduce SavedObjectType to use on AssetReference * Fix Test * Update install.ts Use new `dataTypes` const which replaced `DataType` enum * Update install.ts Remove unused `indexPatternTypes` from outer scope * Update install.ts fix (?) bad updates from before where new/correct value was used but result wasn't exported * Update install.ts * Update install.ts Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: John Schulz <john.schulz@elastic.co>
1 parent 92100f2 commit 1cd477a

16 files changed

Lines changed: 219 additions & 48 deletions

File tree

‎x-pack/plugins/ingest_manager/common/services/package_to_package_policy.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ describe('Ingest Manager - packageToPackagePolicy', () => {
2525
dashboard: [],
2626
visualization: [],
2727
search: [],
28-
'index-pattern': [],
28+
index_pattern: [],
2929
map: [],
3030
},
3131
},

‎x-pack/plugins/ingest_manager/common/types/models/epm.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,21 @@ export type ServiceName = 'kibana' | 'elasticsearch';
3535
export type AgentAssetType = typeof agentAssetTypes;
3636
export type AssetType = KibanaAssetType | ElasticsearchAssetType | ValueOf<AgentAssetType>;
3737

38+
/*
39+
Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased)
40+
*/
3841
export enum KibanaAssetType {
42+
dashboard = 'dashboard',
43+
visualization = 'visualization',
44+
search = 'search',
45+
indexPattern = 'index_pattern',
46+
map = 'map',
47+
}
48+
49+
/*
50+
Enum of saved object types that are allowed to be installed
51+
*/
52+
export enum KibanaSavedObjectType {
3953
dashboard = 'dashboard',
4054
visualization = 'visualization',
4155
search = 'search',
@@ -271,7 +285,7 @@ export type NotInstalled<T = {}> = T & {
271285
export type AssetReference = KibanaAssetReference | EsAssetReference;
272286

273287
export type KibanaAssetReference = Pick<SavedObjectReference, 'id'> & {
274-
type: KibanaAssetType;
288+
type: KibanaSavedObjectType;
275289
};
276290
export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
277291
type: ElasticsearchAssetType;

‎x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export const AssetTitleMap: Record<AssetType, string> = {
2020
ilm_policy: 'ILM Policy',
2121
ingest_pipeline: 'Ingest Pipeline',
2222
transform: 'Transform',
23-
'index-pattern': 'Index Pattern',
23+
index_pattern: 'Index Pattern',
2424
index_template: 'Index Template',
2525
component_template: 'Component Template',
2626
search: 'Saved Search',
@@ -36,7 +36,7 @@ export const ServiceTitleMap: Record<ServiceName, string> = {
3636

3737
export const AssetIcons: Record<KibanaAssetType, IconType> = {
3838
dashboard: 'dashboardApp',
39-
'index-pattern': 'indexPatternApp',
39+
index_pattern: 'indexPatternApp',
4040
search: 'searchProfilerApp',
4141
visualization: 'visualizeApp',
4242
map: 'mapApp',

‎x-pack/plugins/ingest_manager/server/routes/data_streams/handlers.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
*/
66
import { RequestHandler, SavedObjectsClientContract } from 'src/core/server';
77
import { DataStream } from '../../types';
8-
import { GetDataStreamsResponse, KibanaAssetType } from '../../../common';
8+
import { GetDataStreamsResponse, KibanaAssetType, KibanaSavedObjectType } from '../../../common';
99
import { getPackageSavedObjects, getKibanaSavedObject } from '../../services/epm/packages/get';
1010
import { defaultIngestErrorHandler } from '../../errors';
1111

@@ -124,7 +124,7 @@ export const getListHandler: RequestHandler = async (context, request, response)
124124
// then pick the dashboards from the package saved object
125125
const dashboards =
126126
pkgSavedObject[0].attributes?.installed_kibana?.filter(
127-
(o) => o.type === KibanaAssetType.dashboard
127+
(o) => o.type === KibanaSavedObjectType.dashboard
128128
) || [];
129129
// and then pick the human-readable titles from the dashboard saved objects
130130
const enhancedDashboards = await getEnhancedDashboards(

‎x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts‎

Lines changed: 96 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,49 @@ import {
1111
} from 'src/core/server';
1212
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common';
1313
import * as Registry from '../../registry';
14-
import { AssetType, KibanaAssetType, AssetReference } from '../../../../types';
14+
import {
15+
AssetType,
16+
KibanaAssetType,
17+
AssetReference,
18+
AssetParts,
19+
KibanaSavedObjectType,
20+
} from '../../../../types';
1521
import { savedObjectTypes } from '../../packages';
22+
import { indexPatternTypes } from '../index_pattern/install';
1623

1724
type SavedObjectToBe = Required<Pick<SavedObjectsBulkCreateObject, keyof ArchiveAsset>> & {
18-
type: AssetType;
25+
type: KibanaSavedObjectType;
1926
};
2027
export type ArchiveAsset = Pick<
2128
SavedObject,
2229
'id' | 'attributes' | 'migrationVersion' | 'references'
2330
> & {
24-
type: AssetType;
31+
type: KibanaSavedObjectType;
32+
};
33+
34+
// KibanaSavedObjectTypes are used to ensure saved objects being created for a given
35+
// KibanaAssetType have the correct type
36+
const KibanaSavedObjectTypeMapping: Record<KibanaAssetType, KibanaSavedObjectType> = {
37+
[KibanaAssetType.dashboard]: KibanaSavedObjectType.dashboard,
38+
[KibanaAssetType.indexPattern]: KibanaSavedObjectType.indexPattern,
39+
[KibanaAssetType.map]: KibanaSavedObjectType.map,
40+
[KibanaAssetType.search]: KibanaSavedObjectType.search,
41+
[KibanaAssetType.visualization]: KibanaSavedObjectType.visualization,
42+
};
43+
44+
// Define how each asset type will be installed
45+
const AssetInstallers: Record<
46+
KibanaAssetType,
47+
(args: {
48+
savedObjectsClient: SavedObjectsClientContract;
49+
kibanaAssets: ArchiveAsset[];
50+
}) => Promise<Array<SavedObject<unknown>>>
51+
> = {
52+
[KibanaAssetType.dashboard]: installKibanaSavedObjects,
53+
[KibanaAssetType.indexPattern]: installKibanaIndexPatterns,
54+
[KibanaAssetType.map]: installKibanaSavedObjects,
55+
[KibanaAssetType.search]: installKibanaSavedObjects,
56+
[KibanaAssetType.visualization]: installKibanaSavedObjects,
2557
};
2658

2759
export async function getKibanaAsset(key: string): Promise<ArchiveAsset> {
@@ -47,16 +79,22 @@ export function createSavedObjectKibanaAsset(asset: ArchiveAsset): SavedObjectTo
4779
export async function installKibanaAssets(options: {
4880
savedObjectsClient: SavedObjectsClientContract;
4981
pkgName: string;
50-
kibanaAssets: ArchiveAsset[];
82+
kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>;
5183
}): Promise<SavedObject[]> {
5284
const { savedObjectsClient, kibanaAssets } = options;
5385

5486
// install the assets
5587
const kibanaAssetTypes = Object.values(KibanaAssetType);
5688
const installedAssets = await Promise.all(
57-
kibanaAssetTypes.map((assetType) =>
58-
installKibanaSavedObjects({ savedObjectsClient, assetType, kibanaAssets })
59-
)
89+
kibanaAssetTypes.map((assetType) => {
90+
if (kibanaAssets[assetType]) {
91+
return AssetInstallers[assetType]({
92+
savedObjectsClient,
93+
kibanaAssets: kibanaAssets[assetType],
94+
});
95+
}
96+
return [];
97+
})
6098
);
6199
return installedAssets.flat();
62100
}
@@ -74,25 +112,50 @@ export const deleteKibanaInstalledRefs = async (
74112
installed_kibana: installedAssetsToSave,
75113
});
76114
};
77-
export async function getKibanaAssets(paths: string[]) {
78-
const isKibanaAssetType = (path: string) => Registry.pathParts(path).type in KibanaAssetType;
79-
const filteredPaths = paths.filter(isKibanaAssetType);
80-
const kibanaAssets = await Promise.all(filteredPaths.map((path) => getKibanaAsset(path)));
81-
return kibanaAssets;
115+
export async function getKibanaAssets(
116+
paths: string[]
117+
): Promise<Record<KibanaAssetType, ArchiveAsset[]>> {
118+
const kibanaAssetTypes = Object.values(KibanaAssetType);
119+
const isKibanaAssetType = (path: string) => {
120+
const parts = Registry.pathParts(path);
121+
122+
return parts.service === 'kibana' && (kibanaAssetTypes as string[]).includes(parts.type);
123+
};
124+
125+
const filteredPaths = paths
126+
.filter(isKibanaAssetType)
127+
.map<[string, AssetParts]>((path) => [path, Registry.pathParts(path)]);
128+
129+
const assetArrays: Array<Promise<ArchiveAsset[]>> = [];
130+
for (const assetType of kibanaAssetTypes) {
131+
const matching = filteredPaths.filter(([path, parts]) => parts.type === assetType);
132+
133+
assetArrays.push(Promise.all(matching.map(([path]) => path).map(getKibanaAsset)));
134+
}
135+
136+
const resolvedAssets = await Promise.all(assetArrays);
137+
138+
const result = {} as Record<KibanaAssetType, ArchiveAsset[]>;
139+
140+
for (const [index, assetType] of kibanaAssetTypes.entries()) {
141+
const expectedType = KibanaSavedObjectTypeMapping[assetType];
142+
const properlyTypedAssets = resolvedAssets[index].filter(({ type }) => type === expectedType);
143+
144+
result[assetType] = properlyTypedAssets;
145+
}
146+
147+
return result;
82148
}
149+
83150
async function installKibanaSavedObjects({
84151
savedObjectsClient,
85-
assetType,
86152
kibanaAssets,
87153
}: {
88154
savedObjectsClient: SavedObjectsClientContract;
89-
assetType: KibanaAssetType;
90155
kibanaAssets: ArchiveAsset[];
91156
}) {
92-
const isSameType = (asset: ArchiveAsset) => assetType === asset.type;
93-
const filteredKibanaAssets = kibanaAssets.filter((asset) => isSameType(asset));
94157
const toBeSavedObjects = await Promise.all(
95-
filteredKibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset))
158+
kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset))
96159
);
97160

98161
if (toBeSavedObjects.length === 0) {
@@ -105,8 +168,23 @@ async function installKibanaSavedObjects({
105168
}
106169
}
107170

171+
async function installKibanaIndexPatterns({
172+
savedObjectsClient,
173+
kibanaAssets,
174+
}: {
175+
savedObjectsClient: SavedObjectsClientContract;
176+
kibanaAssets: ArchiveAsset[];
177+
}) {
178+
// Filter out any reserved index patterns
179+
const reservedPatterns = indexPatternTypes.map((pattern) => `${pattern}-*`);
180+
181+
const nonReservedPatterns = kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id));
182+
183+
return installKibanaSavedObjects({ savedObjectsClient, kibanaAssets: nonReservedPatterns });
184+
}
185+
108186
export function toAssetReference({ id, type }: SavedObject) {
109-
const reference: AssetReference = { id, type: type as KibanaAssetType };
187+
const reference: AssetReference = { id, type: type as KibanaSavedObjectType };
110188

111189
return reference;
112190
}

‎x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ export interface IndexPatternField {
7272
readFromDocValues: boolean;
7373
}
7474

75+
export const indexPatternTypes = Object.values(dataTypes);
7576
// TODO: use a function overload and make pkgName and pkgVersion required for install/update
7677
// and not for an update removal. or separate out the functions
7778
export async function installIndexPatterns(
@@ -116,7 +117,6 @@ export async function installIndexPatterns(
116117
const packageVersionsInfo = await Promise.all(packageVersionsFetchInfoPromise);
117118

118119
// for each index pattern type, create an index pattern
119-
const indexPatternTypes = Object.values(dataTypes);
120120
indexPatternTypes.forEach(async (indexPatternType) => {
121121
// if this is an update because a package is being uninstalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
122122
if (!pkgName && installedPackages.length === 0) {

‎x-pack/plugins/ingest_manager/server/services/epm/packages/ensure_installed_default_packages.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import { ElasticsearchAssetType, Installation, KibanaAssetType } from '../../../types';
7+
import { ElasticsearchAssetType, Installation, KibanaSavedObjectType } from '../../../types';
88
import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
99

1010
jest.mock('./install');
@@ -41,7 +41,7 @@ const mockInstallation: SavedObject<Installation> = {
4141
type: 'epm-packages',
4242
attributes: {
4343
id: 'test-pkg',
44-
installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
44+
installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
4545
installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
4646
es_index_patterns: { pattern: 'pattern-name' },
4747
name: 'test package',

‎x-pack/plugins/ingest_manager/server/services/epm/packages/get_install_type.test.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66
import { SavedObject } from 'src/core/server';
7-
import { ElasticsearchAssetType, Installation, KibanaAssetType } from '../../../types';
7+
import { ElasticsearchAssetType, Installation, KibanaSavedObjectType } from '../../../types';
88
import { getInstallType } from './install';
99

1010
const mockInstallation: SavedObject<Installation> = {
@@ -13,7 +13,7 @@ const mockInstallation: SavedObject<Installation> = {
1313
type: 'epm-packages',
1414
attributes: {
1515
id: 'test-pkg',
16-
installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
16+
installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
1717
installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
1818
es_index_patterns: { pattern: 'pattern-name' },
1919
name: 'test packagek',
@@ -30,7 +30,7 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
3030
type: 'epm-packages',
3131
attributes: {
3232
id: 'test-pkg',
33-
installed_kibana: [{ type: KibanaAssetType.dashboard, id: 'dashboard-1' }],
33+
installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
3434
installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
3535
es_index_patterns: { pattern: 'pattern-name' },
3636
name: 'test packagek',

‎x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
KibanaAssetReference,
1919
EsAssetReference,
2020
InstallType,
21+
KibanaAssetType,
2122
} from '../../../types';
2223
import * as Registry from '../registry';
2324
import {
@@ -364,9 +365,9 @@ export async function createInstallation(options: {
364365
export const saveKibanaAssetsRefs = async (
365366
savedObjectsClient: SavedObjectsClientContract,
366367
pkgName: string,
367-
kibanaAssets: ArchiveAsset[]
368+
kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>
368369
) => {
369-
const assetRefs = kibanaAssets.map(toAssetReference);
370+
const assetRefs = Object.values(kibanaAssets).flat().map(toAssetReference);
370371
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
371372
installed_kibana: assetRefs,
372373
});

‎x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@ import {
1212
AssetType,
1313
CallESAsCurrentUser,
1414
ElasticsearchAssetType,
15+
EsAssetReference,
16+
KibanaAssetReference,
17+
Installation,
1518
} from '../../../types';
1619
import { getInstallation, savedObjectTypes } from './index';
1720
import { deletePipeline } from '../elasticsearch/ingest_pipeline/';
@@ -46,7 +49,7 @@ export async function removeInstallation(options: {
4649

4750
// Delete the installed assets
4851
const installedAssets = [...installation.installed_kibana, ...installation.installed_es];
49-
await deleteAssets(installedAssets, savedObjectsClient, callCluster);
52+
await deleteAssets(installation, savedObjectsClient, callCluster);
5053

5154
// Delete the manager saved object with references to the asset objects
5255
// could also update with [] or some other state
@@ -64,26 +67,43 @@ export async function removeInstallation(options: {
6467
// successful delete's in SO client return {}. return something more useful
6568
return installedAssets;
6669
}
67-
async function deleteAssets(
68-
installedObjects: AssetReference[],
69-
savedObjectsClient: SavedObjectsClientContract,
70-
callCluster: CallESAsCurrentUser
70+
71+
function deleteKibanaAssets(
72+
installedObjects: KibanaAssetReference[],
73+
savedObjectsClient: SavedObjectsClientContract
7174
) {
72-
const logger = appContextService.getLogger();
73-
const deletePromises = installedObjects.map(async ({ id, type }) => {
75+
return installedObjects.map(async ({ id, type }) => {
76+
return savedObjectsClient.delete(type, id);
77+
});
78+
}
79+
80+
function deleteESAssets(installedObjects: EsAssetReference[], callCluster: CallESAsCurrentUser) {
81+
return installedObjects.map(async ({ id, type }) => {
7482
const assetType = type as AssetType;
75-
if (savedObjectTypes.includes(assetType)) {
76-
return savedObjectsClient.delete(assetType, id);
77-
} else if (assetType === ElasticsearchAssetType.ingestPipeline) {
83+
if (assetType === ElasticsearchAssetType.ingestPipeline) {
7884
return deletePipeline(callCluster, id);
7985
} else if (assetType === ElasticsearchAssetType.indexTemplate) {
8086
return deleteTemplate(callCluster, id);
8187
} else if (assetType === ElasticsearchAssetType.transform) {
8288
return deleteTransforms(callCluster, [id]);
8389
}
8490
});
91+
}
92+
93+
async function deleteAssets(
94+
{ installed_es: installedEs, installed_kibana: installedKibana }: Installation,
95+
savedObjectsClient: SavedObjectsClientContract,
96+
callCluster: CallESAsCurrentUser
97+
) {
98+
const logger = appContextService.getLogger();
99+
100+
const deletePromises: Array<Promise<unknown>> = [
101+
...deleteESAssets(installedEs, callCluster),
102+
...deleteKibanaAssets(installedKibana, savedObjectsClient),
103+
];
104+
85105
try {
86-
await Promise.all([...deletePromises]);
106+
await Promise.all(deletePromises);
87107
} catch (err) {
88108
logger.error(err);
89109
}

0 commit comments

Comments
 (0)