Skip to content

Commit fc41fa7

Browse files
neptunianneptunian
committed
update current write indices
1 parent c8c03bb commit fc41fa7

6 files changed

Lines changed: 226 additions & 104 deletions

File tree

‎x-pack/plugins/ingest_manager/common/types/models/epm.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,10 +256,32 @@ export enum DefaultPackages {
256256
endpoint = 'endpoint',
257257
}
258258

259+
export interface Mappings {
260+
dynamic?: boolean | 'strict';
261+
date_detection?: boolean;
262+
dynamic_templates?: any[];
263+
properties: Record<string, Mapping>;
264+
}
265+
266+
interface Mapping {
267+
type: string;
268+
properties?: Record<string, Mapping>;
269+
}
270+
259271
export interface IndexTemplate {
260272
order: number;
261273
index_patterns: string[];
262274
settings: any;
263-
mappings: object;
275+
mappings: Mappings;
264276
aliases: object;
265277
}
278+
279+
export interface TemplateRef {
280+
templateName: string;
281+
indexTemplate: IndexTemplate;
282+
}
283+
284+
export interface CurrentIndex {
285+
indexName: string;
286+
indexTemplate: IndexTemplate;
287+
}

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap‎

Lines changed: 0 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,7 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import {
8-
AssetReference,
9-
Dataset,
10-
RegistryPackage,
11-
IngestAssetType,
12-
ElasticsearchAssetType,
13-
} from '../../../../types';
7+
import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types';
148
import { CallESAsCurrentUser } from '../../../../types';
159
import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
1610
import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
@@ -22,15 +16,15 @@ export const installTemplates = async (
2216
callCluster: CallESAsCurrentUser,
2317
pkgName: string,
2418
pkgVersion: string
25-
) => {
19+
): Promise<TemplateRef[]> => {
2620
// install any pre-built index template assets,
2721
// atm, this is only the base package's global template
2822
installPreBuiltTemplates(pkgName, pkgVersion, callCluster);
2923

3024
// build templates per dataset from yml files
3125
const datasets = registryPackage.datasets;
3226
if (datasets) {
33-
const templates = datasets.reduce<Array<Promise<AssetReference>>>((acc, dataset) => {
27+
const installTemplatePromises = datasets.reduce<Array<Promise<TemplateRef>>>((acc, dataset) => {
3428
acc.push(
3529
installTemplateForDataset({
3630
pkg: registryPackage,
@@ -40,7 +34,9 @@ export const installTemplates = async (
4034
);
4135
return acc;
4236
}, []);
43-
return Promise.all(templates).then(results => results.flat());
37+
38+
const res = await Promise.all(installTemplatePromises);
39+
return res.flat();
4440
}
4541
return [];
4642
};
@@ -84,7 +80,7 @@ export async function installTemplateForDataset({
8480
pkg: RegistryPackage;
8581
callCluster: CallESAsCurrentUser;
8682
dataset: Dataset;
87-
}): Promise<AssetReference> {
83+
}): Promise<TemplateRef> {
8884
const fields = await loadFieldsFromYaml(pkg, dataset.path);
8985
return installTemplate({
9086
callCluster,
@@ -104,7 +100,7 @@ export async function installTemplate({
104100
fields: Field[];
105101
dataset: Dataset;
106102
packageVersion: string;
107-
}): Promise<AssetReference> {
103+
}): Promise<TemplateRef> {
108104
const mappings = generateMappings(processFields(fields));
109105
const templateName = generateTemplateName(dataset);
110106
let pipelineName;
@@ -122,6 +118,8 @@ export async function installTemplate({
122118
body: template,
123119
});
124120

125-
// The id of a template is its name
126-
return { id: templateName, type: IngestAssetType.IndexTemplate };
121+
return {
122+
templateName,
123+
indexTemplate: template,
124+
};
127125
}

‎x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts‎

Lines changed: 100 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,13 @@
55
*/
66

77
import { Field, Fields } from '../../fields/field';
8-
import { Dataset, IndexTemplate } from '../../../../types';
8+
import {
9+
Dataset,
10+
IndexTemplate,
11+
CallESAsCurrentUser,
12+
TemplateRef,
13+
CurrentIndex,
14+
} from '../../../../types';
915
import { getDatasetAssetBaseName } from '../index';
1016

1117
interface Properties {
@@ -235,9 +241,10 @@ function getBaseTemplate(type: string, templateName: string, mappings: Mappings)
235241
},
236242
mappings: {
237243
// To be filled with interesting information about this specific index
244+
/*
238245
_meta: {
239246
package: 'foo',
240-
},
247+
},*/
241248
// All the dynamic field mappings
242249
dynamic_templates: [
243250
// This makes sure all mappings are keywords by default
@@ -261,3 +268,94 @@ function getBaseTemplate(type: string, templateName: string, mappings: Mappings)
261268
aliases: {},
262269
};
263270
}
271+
272+
export const updateCurrentWriteIndices = async (
273+
callCluster: CallESAsCurrentUser,
274+
templates: TemplateRef[]
275+
): Promise<void> => {
276+
if (!templates) return;
277+
278+
const allIndices = await queryIndicesFromTemplates(callCluster, templates);
279+
return updateAllIndices(allIndices, callCluster);
280+
};
281+
282+
const queryIndicesFromTemplates = async (
283+
callCluster: CallESAsCurrentUser,
284+
templates: TemplateRef[]
285+
): Promise<CurrentIndex[]> => {
286+
const indexPromises = templates.map(template => {
287+
return createIndexFromNamespace(callCluster, template);
288+
});
289+
const indexObjects = await Promise.all(indexPromises);
290+
return indexObjects.filter(item => item !== undefined).flat();
291+
};
292+
293+
const createIndexFromNamespace = async (
294+
callCluster: CallESAsCurrentUser,
295+
template: TemplateRef
296+
): Promise<CurrentIndex[] | undefined> => {
297+
const { templateName, indexTemplate } = template;
298+
const res = await callCluster('search', getIndexByNamespaceQuery(templateName));
299+
const namespaces: any[] = res?.aggregations?.streams.buckets;
300+
if (namespaces) {
301+
return namespaces.map(namespace => ({
302+
indexName: `${templateName}-${namespace.key}`,
303+
indexTemplate,
304+
}));
305+
}
306+
};
307+
308+
const updateAllIndices = async (
309+
indexNameWithTemplates: CurrentIndex[],
310+
callCluster: CallESAsCurrentUser
311+
): Promise<void> => {
312+
const updateIndexPromises = indexNameWithTemplates.map(({ indexName, indexTemplate }) => {
313+
return updateExistingIndex({ indexName, callCluster, indexTemplate });
314+
});
315+
await Promise.all(updateIndexPromises);
316+
};
317+
const updateExistingIndex = async ({
318+
indexName,
319+
callCluster,
320+
indexTemplate,
321+
}: {
322+
indexName: string;
323+
callCluster: CallESAsCurrentUser;
324+
indexTemplate: IndexTemplate;
325+
}) => {
326+
const { settings, mappings } = indexTemplate;
327+
// try to update the mappings first
328+
// for now we assume updates are compatible
329+
try {
330+
await callCluster('indices.putMapping', {
331+
index: indexName,
332+
body: mappings,
333+
});
334+
} catch (err) {
335+
throw new Error('incompatible mappings update');
336+
}
337+
// update settings after mappings was successful to ensure
338+
// pointing to theme new pipeline is safe
339+
// for now, only update the pipeline
340+
if (!settings.index.default_pipeline) return;
341+
try {
342+
await callCluster('indices.putSettings', {
343+
index: indexName,
344+
body: { index: { default_pipeline: settings.index.default_pipeline } },
345+
});
346+
} catch (err) {
347+
throw new Error('incompatible settings update');
348+
}
349+
};
350+
351+
const getIndexByNamespaceQuery = (templateName: string) => ({
352+
index: `${templateName}-*`,
353+
size: 1,
354+
body: {
355+
aggs: {
356+
streams: {
357+
terms: { field: 'fields.stream.namespace' },
358+
},
359+
},
360+
},
361+
});

0 commit comments

Comments
 (0)