Skip to content

Commit cb5deee

Browse files
jonathan-buttnerelasticmachineskh
committed
[EPM][Security Solution] Implementing dataset component templates (#70517)
* Implementing dataset component templates * Fixing test * Temporary fix to include timestamp with any component template created * Update package registry docker image for CI. * Adapt to new registry filesystem layout. * Adjust tests to changed registry behavior. * Adding a test for mappings and settings overrides * Wrap all the tests in the docker check Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: Sonja Krause-Harder <sonja.krause-harder@elastic.co>
1 parent 7cfe8ba commit cb5deee

15 files changed

Lines changed: 298 additions & 4 deletions

File tree

‎x-pack/plugins/ingest_manager/common/types/models/epm.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,12 @@ export interface Dataset {
175175
package: string;
176176
path: string;
177177
ingest_pipeline: string;
178+
elasticsearch?: RegistryElasticsearch;
179+
}
180+
181+
export interface RegistryElasticsearch {
182+
'index_template.settings'?: object;
183+
'index_template.mappings'?: object;
178184
}
179185

180186
// EPR types this as `[]map[string]interface{}`
@@ -272,6 +278,7 @@ export interface IndexTemplate {
272278
data_stream: {
273279
timestamp_field: string;
274280
};
281+
composed_of: string[];
275282
_meta: object;
276283
}
277284

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts‎

Lines changed: 108 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,13 @@
55
*/
66

77
import Boom from 'boom';
8-
import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types';
8+
import {
9+
Dataset,
10+
RegistryPackage,
11+
ElasticsearchAssetType,
12+
TemplateRef,
13+
RegistryElasticsearch,
14+
} from '../../../../types';
915
import { CallESAsCurrentUser } from '../../../../types';
1016
import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
1117
import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
@@ -157,6 +163,98 @@ export async function installTemplateForDataset({
157163
});
158164
}
159165

166+
function putComponentTemplate(
167+
body: object | undefined,
168+
name: string,
169+
callCluster: CallESAsCurrentUser
170+
): { clusterPromise: Promise<any>; name: string } | undefined {
171+
if (body) {
172+
const callClusterParams: {
173+
method: string;
174+
path: string;
175+
ignore: number[];
176+
body: any;
177+
} = {
178+
method: 'PUT',
179+
path: `/_component_template/${name}`,
180+
ignore: [404],
181+
body,
182+
};
183+
184+
return { clusterPromise: callCluster('transport.request', callClusterParams), name };
185+
}
186+
}
187+
188+
function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) {
189+
let mappingsTemplate;
190+
let settingsTemplate;
191+
192+
if (registryElasticsearch && registryElasticsearch['index_template.mappings']) {
193+
mappingsTemplate = {
194+
template: {
195+
mappings: {
196+
...registryElasticsearch['index_template.mappings'],
197+
// temporary change until https://github.com/elastic/elasticsearch/issues/58956 is resolved
198+
// hopefully we'll be able to remove the entire properties section once that issue is resolved
199+
properties: {
200+
// if the timestamp_field changes here: https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts#L309
201+
// we'll need to update this as well
202+
'@timestamp': {
203+
type: 'date',
204+
},
205+
},
206+
},
207+
},
208+
};
209+
}
210+
211+
if (registryElasticsearch && registryElasticsearch['index_template.settings']) {
212+
settingsTemplate = {
213+
template: {
214+
settings: registryElasticsearch['index_template.settings'],
215+
},
216+
};
217+
}
218+
return { settingsTemplate, mappingsTemplate };
219+
}
220+
221+
async function installDatasetComponentTemplates(
222+
templateName: string,
223+
registryElasticsearch: RegistryElasticsearch | undefined,
224+
callCluster: CallESAsCurrentUser
225+
) {
226+
const templates: string[] = [];
227+
const componentPromises: Array<Promise<any>> = [];
228+
229+
const compTemplates = buildComponentTemplates(registryElasticsearch);
230+
231+
const mappings = putComponentTemplate(
232+
compTemplates.mappingsTemplate,
233+
`${templateName}-mappings`,
234+
callCluster
235+
);
236+
237+
const settings = putComponentTemplate(
238+
compTemplates.settingsTemplate,
239+
`${templateName}-settings`,
240+
callCluster
241+
);
242+
243+
if (mappings) {
244+
templates.push(mappings.name);
245+
componentPromises.push(mappings.clusterPromise);
246+
}
247+
248+
if (settings) {
249+
templates.push(settings.name);
250+
componentPromises.push(settings.clusterPromise);
251+
}
252+
253+
// TODO: Check return values for errors
254+
await Promise.all(componentPromises);
255+
return templates;
256+
}
257+
160258
export async function installTemplate({
161259
callCluster,
162260
fields,
@@ -180,13 +278,22 @@ export async function installTemplate({
180278
packageVersion,
181279
});
182280
}
281+
282+
const composedOfTemplates = await installDatasetComponentTemplates(
283+
templateName,
284+
dataset.elasticsearch,
285+
callCluster
286+
);
287+
183288
const template = getTemplate({
184289
type: dataset.type,
185290
templateName,
186291
mappings,
187292
pipelineName,
188293
packageName,
294+
composedOfTemplates,
189295
});
296+
190297
// TODO: Check return values for errors
191298
const callClusterParams: {
192299
method: string;

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,37 @@ test('get template', () => {
2929
templateName,
3030
packageName: 'nginx',
3131
mappings: { properties: {} },
32+
composedOfTemplates: [],
3233
});
3334
expect(template.index_patterns).toStrictEqual([`${templateName}-*`]);
3435
});
3536

37+
test('adds composed_of correctly', () => {
38+
const composedOfTemplates = ['component1', 'component2'];
39+
40+
const template = getTemplate({
41+
type: 'logs',
42+
templateName: 'name',
43+
packageName: 'nginx',
44+
mappings: { properties: {} },
45+
composedOfTemplates,
46+
});
47+
expect(template.composed_of).toStrictEqual(composedOfTemplates);
48+
});
49+
50+
test('adds empty composed_of correctly', () => {
51+
const composedOfTemplates: string[] = [];
52+
53+
const template = getTemplate({
54+
type: 'logs',
55+
templateName: 'name',
56+
packageName: 'nginx',
57+
mappings: { properties: {} },
58+
composedOfTemplates,
59+
});
60+
expect(template.composed_of).toStrictEqual(composedOfTemplates);
61+
});
62+
3663
test('tests loading base.yml', () => {
3764
const ymlPath = path.join(__dirname, '../../fields/tests/base.yml');
3865
const fieldsYML = readFileSync(ymlPath, 'utf-8');
@@ -45,6 +72,7 @@ test('tests loading base.yml', () => {
4572
templateName: 'foo',
4673
packageName: 'nginx',
4774
mappings,
75+
composedOfTemplates: [],
4876
});
4977

5078
expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -62,6 +90,7 @@ test('tests loading coredns.logs.yml', () => {
6290
templateName: 'foo',
6391
packageName: 'coredns',
6492
mappings,
93+
composedOfTemplates: [],
6594
});
6695

6796
expect(template).toMatchSnapshot(path.basename(ymlPath));
@@ -79,6 +108,7 @@ test('tests loading system.yml', () => {
79108
templateName: 'whatsthis',
80109
packageName: 'system',
81110
mappings,
111+
composedOfTemplates: [],
82112
});
83113

84114
expect(template).toMatchSnapshot(path.basename(ymlPath));

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,16 @@ export function getTemplate({
4343
mappings,
4444
pipelineName,
4545
packageName,
46+
composedOfTemplates,
4647
}: {
4748
type: string;
4849
templateName: string;
4950
mappings: IndexTemplateMappings;
5051
pipelineName?: string | undefined;
5152
packageName: string;
53+
composedOfTemplates: string[];
5254
}): IndexTemplate {
53-
const template = getBaseTemplate(type, templateName, mappings, packageName);
55+
const template = getBaseTemplate(type, templateName, mappings, packageName, composedOfTemplates);
5456
if (pipelineName) {
5557
template.template.settings.index.default_pipeline = pipelineName;
5658
}
@@ -244,7 +246,8 @@ function getBaseTemplate(
244246
type: string,
245247
templateName: string,
246248
mappings: IndexTemplateMappings,
247-
packageName: string
249+
packageName: string,
250+
composedOfTemplates: string[]
248251
): IndexTemplate {
249252
return {
250253
// This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*)
@@ -308,6 +311,7 @@ function getBaseTemplate(
308311
data_stream: {
309312
timestamp_field: '@timestamp',
310313
},
314+
composed_of: composedOfTemplates,
311315
_meta: {
312316
package: {
313317
name: packageName,

‎x-pack/plugins/ingest_manager/server/types/index.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ export {
4141
PackageInfo,
4242
RegistryVarsEntry,
4343
Dataset,
44+
RegistryElasticsearch,
4445
AssetReference,
4546
ElasticsearchAssetType,
4647
IngestAssetType,

‎x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
- name: dataset.type
2+
type: constant_keyword
3+
description: >
4+
Dataset type.
5+
- name: dataset.name
6+
type: constant_keyword
7+
description: >
8+
Dataset name.
9+
- name: dataset.namespace
10+
type: constant_keyword
11+
description: >
12+
Dataset namespace.
13+
- name: '@timestamp'
14+
type: date
15+
description: >
16+
Event timestamp.

‎x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
title: Test Dataset
2+
3+
type: logs
4+
5+
elasticsearch:
6+
index_template.mappings:
7+
dynamic: false
8+
index_template.settings:
9+
index.lifecycle.name: reference

‎x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Test package
2+
3+
For testing the that the settings and mappings section get used

‎x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg‎

Lines changed: 7 additions & 0 deletions
Loading

0 commit comments

Comments
 (0)