Skip to content

Commit 02448ef

Browse files
jsorianojsoriano
committed
Fix generation of dynamic mapping for object with specific subfield (#204104)
Fix generation of dynamic mapping for objects that have more specific subfields in separate definitions. This can be reproduced for example with: ``` - name: labels type: object object_type: keyword object_type_mapping_type: '*' - name: labels.count type: long ``` Fleet expands and deduplicates field definitions before generating the mappings, so the definitions above are converted to something like the following: ``` - name: labels type: group object_type: keyword object_type_mapping_type: '*' fields: - name: count type: long ``` Usually fields of type `group` don't have an `object_type`, so this was being ignored, the dynamic mapping was not being generated. This issue was not reproduced if the object field name includes a wildcard, like in `labels.*`, because then the expansion and deduplication resolves to something like this: ``` - name: labels type: group fields: - name: '*' type: object object_type: keyword object_type_mapping_type: '*' - name: count type: long ``` (cherry picked from commit e3877e0)
1 parent 5b9434c commit 02448ef

2 files changed

Lines changed: 203 additions & 117 deletions

File tree

‎x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -786,6 +786,84 @@ describe('EPM template', () => {
786786
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
787787
});
788788

789+
it('tests processing object field with more specific properties without wildcard', () => {
790+
const objectFieldWithPropertyReversedLiteralYml = `
791+
- name: labels
792+
type: object
793+
object_type: keyword
794+
object_type_mapping_type: '*'
795+
- name: labels.count
796+
type: long
797+
`;
798+
const objectFieldWithPropertyReversedMapping = {
799+
dynamic_templates: [
800+
{
801+
labels: {
802+
path_match: 'labels.*',
803+
match_mapping_type: '*',
804+
mapping: {
805+
type: 'keyword',
806+
},
807+
},
808+
},
809+
],
810+
properties: {
811+
labels: {
812+
dynamic: true,
813+
type: 'object',
814+
properties: {
815+
count: {
816+
type: 'long',
817+
},
818+
},
819+
},
820+
},
821+
};
822+
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
823+
const processedFields = processFields(fields);
824+
const mappings = generateMappings(processedFields);
825+
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
826+
});
827+
828+
it('tests processing object field with more specific properties with wildcard', () => {
829+
const objectFieldWithPropertyReversedLiteralYml = `
830+
- name: labels.*
831+
type: object
832+
object_type: keyword
833+
object_type_mapping_type: '*'
834+
- name: labels.count
835+
type: long
836+
`;
837+
const objectFieldWithPropertyReversedMapping = {
838+
dynamic_templates: [
839+
{
840+
'labels.*': {
841+
path_match: 'labels.*',
842+
match_mapping_type: '*',
843+
mapping: {
844+
type: 'keyword',
845+
},
846+
},
847+
},
848+
],
849+
properties: {
850+
labels: {
851+
dynamic: true,
852+
type: 'object',
853+
properties: {
854+
count: {
855+
type: 'long',
856+
},
857+
},
858+
},
859+
},
860+
};
861+
const fields: Field[] = load(objectFieldWithPropertyReversedLiteralYml);
862+
const processedFields = processFields(fields);
863+
const mappings = generateMappings(processedFields);
864+
expect(mappings).toEqual(objectFieldWithPropertyReversedMapping);
865+
});
866+
789867
it('tests processing object field with subobjects set to false (case B)', () => {
790868
const objectFieldWithPropertyReversedLiteralYml = `
791869
- name: b.labels.*

‎x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts‎

Lines changed: 125 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,124 @@ function _generateMappings(
298298
}
299299
}
300300

301+
function addObjectAsDynamicMapping(field: Field) {
302+
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
303+
const pathMatch = path.includes('*') ? path : `${path}.*`;
304+
305+
let dynProperties: Properties = getDefaultProperties(field);
306+
let matchingType: string | undefined;
307+
switch (field.object_type) {
308+
case 'histogram':
309+
dynProperties = histogram(field);
310+
matchingType = field.object_type_mapping_type ?? '*';
311+
break;
312+
case 'ip':
313+
case 'keyword':
314+
case 'match_only_text':
315+
case 'text':
316+
case 'wildcard':
317+
dynProperties.type = field.object_type;
318+
matchingType = field.object_type_mapping_type ?? 'string';
319+
break;
320+
case 'scaled_float':
321+
dynProperties = scaledFloat(field);
322+
matchingType = field.object_type_mapping_type ?? '*';
323+
break;
324+
case 'aggregate_metric_double':
325+
dynProperties.type = field.object_type;
326+
dynProperties.metrics = field.metrics;
327+
dynProperties.default_metric = field.default_metric;
328+
matchingType = field.object_type_mapping_type ?? '*';
329+
break;
330+
case 'double':
331+
case 'float':
332+
case 'half_float':
333+
dynProperties.type = field.object_type;
334+
if (isIndexModeTimeSeries) {
335+
dynProperties.time_series_metric = field.metric_type;
336+
}
337+
matchingType = field.object_type_mapping_type ?? 'double';
338+
break;
339+
case 'byte':
340+
case 'long':
341+
case 'short':
342+
case 'unsigned_long':
343+
dynProperties.type = field.object_type;
344+
if (isIndexModeTimeSeries) {
345+
dynProperties.time_series_metric = field.metric_type;
346+
}
347+
matchingType = field.object_type_mapping_type ?? 'long';
348+
break;
349+
case 'integer':
350+
// Map integers as long, as in other cases.
351+
dynProperties.type = 'long';
352+
if (isIndexModeTimeSeries) {
353+
dynProperties.time_series_metric = field.metric_type;
354+
}
355+
matchingType = field.object_type_mapping_type ?? 'long';
356+
break;
357+
case 'boolean':
358+
dynProperties.type = field.object_type;
359+
if (isIndexModeTimeSeries) {
360+
dynProperties.time_series_metric = field.metric_type;
361+
}
362+
matchingType = field.object_type_mapping_type ?? field.object_type;
363+
break;
364+
case 'group':
365+
if (!field?.fields) {
366+
break;
367+
}
368+
const subFields = field.fields.map((subField) => ({
369+
...subField,
370+
type: 'object',
371+
object_type: subField.object_type ?? subField.type,
372+
}));
373+
const mappings = _generateMappings(
374+
subFields,
375+
{
376+
...ctx,
377+
groupFieldName: ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name,
378+
},
379+
isIndexModeTimeSeries
380+
);
381+
if (mappings.hasDynamicTemplateMappings) {
382+
hasDynamicTemplateMappings = true;
383+
}
384+
break;
385+
case 'flattened':
386+
dynProperties.type = field.object_type;
387+
matchingType = field.object_type_mapping_type ?? 'object';
388+
break;
389+
default:
390+
throw new PackageInvalidArchiveError(
391+
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
392+
);
393+
}
394+
395+
if (field.dimension && isIndexModeTimeSeries) {
396+
dynProperties.time_series_dimension = field.dimension;
397+
}
398+
399+
// When a wildcard field specifies the subobjects setting,
400+
// the parent intermediate object should set the subobjects
401+
// setting.
402+
//
403+
// For example, if a wildcard field `foo.*` has subobjects,
404+
// we should set subobjects on the intermediate object `foo`.
405+
//
406+
if (field.subobjects !== undefined && path.includes('*')) {
407+
subobjects = field.subobjects;
408+
}
409+
410+
if (dynProperties && matchingType) {
411+
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
412+
413+
// Add the parent object as static property, this is needed for
414+
// index templates not using `"dynamic": true`.
415+
addParentObjectAsStaticProperty(field);
416+
}
417+
}
418+
301419
// TODO: this can happen when the fields property in fields.yml is present but empty
302420
// Maybe validation should be moved to fields/field.ts
303421
if (fields) {
@@ -359,123 +477,7 @@ function _generateMappings(
359477
}
360478

361479
if (type === 'object' && field.object_type) {
362-
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
363-
const pathMatch = path.includes('*') ? path : `${path}.*`;
364-
365-
let dynProperties: Properties = getDefaultProperties(field);
366-
let matchingType: string | undefined;
367-
switch (field.object_type) {
368-
case 'histogram':
369-
dynProperties = histogram(field);
370-
matchingType = field.object_type_mapping_type ?? '*';
371-
break;
372-
case 'ip':
373-
case 'keyword':
374-
case 'match_only_text':
375-
case 'text':
376-
case 'wildcard':
377-
dynProperties.type = field.object_type;
378-
matchingType = field.object_type_mapping_type ?? 'string';
379-
break;
380-
case 'scaled_float':
381-
dynProperties = scaledFloat(field);
382-
matchingType = field.object_type_mapping_type ?? '*';
383-
break;
384-
case 'aggregate_metric_double':
385-
dynProperties.type = field.object_type;
386-
dynProperties.metrics = field.metrics;
387-
dynProperties.default_metric = field.default_metric;
388-
matchingType = field.object_type_mapping_type ?? '*';
389-
break;
390-
case 'double':
391-
case 'float':
392-
case 'half_float':
393-
dynProperties.type = field.object_type;
394-
if (isIndexModeTimeSeries) {
395-
dynProperties.time_series_metric = field.metric_type;
396-
}
397-
matchingType = field.object_type_mapping_type ?? 'double';
398-
break;
399-
case 'byte':
400-
case 'long':
401-
case 'short':
402-
case 'unsigned_long':
403-
dynProperties.type = field.object_type;
404-
if (isIndexModeTimeSeries) {
405-
dynProperties.time_series_metric = field.metric_type;
406-
}
407-
matchingType = field.object_type_mapping_type ?? 'long';
408-
break;
409-
case 'integer':
410-
// Map integers as long, as in other cases.
411-
dynProperties.type = 'long';
412-
if (isIndexModeTimeSeries) {
413-
dynProperties.time_series_metric = field.metric_type;
414-
}
415-
matchingType = field.object_type_mapping_type ?? 'long';
416-
break;
417-
case 'boolean':
418-
dynProperties.type = field.object_type;
419-
if (isIndexModeTimeSeries) {
420-
dynProperties.time_series_metric = field.metric_type;
421-
}
422-
matchingType = field.object_type_mapping_type ?? field.object_type;
423-
break;
424-
case 'group':
425-
if (!field?.fields) {
426-
break;
427-
}
428-
const subFields = field.fields.map((subField) => ({
429-
...subField,
430-
type: 'object',
431-
object_type: subField.object_type ?? subField.type,
432-
}));
433-
const mappings = _generateMappings(
434-
subFields,
435-
{
436-
...ctx,
437-
groupFieldName: ctx.groupFieldName
438-
? `${ctx.groupFieldName}.${field.name}`
439-
: field.name,
440-
},
441-
isIndexModeTimeSeries
442-
);
443-
if (mappings.hasDynamicTemplateMappings) {
444-
hasDynamicTemplateMappings = true;
445-
}
446-
break;
447-
case 'flattened':
448-
dynProperties.type = field.object_type;
449-
matchingType = field.object_type_mapping_type ?? 'object';
450-
break;
451-
default:
452-
throw new PackageInvalidArchiveError(
453-
`No dynamic mapping generated for field ${path} of type ${field.object_type}`
454-
);
455-
}
456-
457-
if (field.dimension && isIndexModeTimeSeries) {
458-
dynProperties.time_series_dimension = field.dimension;
459-
}
460-
461-
// When a wildcard field specifies the subobjects setting,
462-
// the parent intermediate object should set the subobjects
463-
// setting.
464-
//
465-
// For example, if a wildcard field `foo.*` has subobjects,
466-
// we should set subobjects on the intermediate object `foo`.
467-
//
468-
if (field.subobjects !== undefined && path.includes('*')) {
469-
subobjects = field.subobjects;
470-
}
471-
472-
if (dynProperties && matchingType) {
473-
addDynamicMappingWithIntermediateObjects(path, pathMatch, matchingType, dynProperties);
474-
475-
// Add the parent object as static property, this is needed for
476-
// index templates not using `"dynamic": true`.
477-
addParentObjectAsStaticProperty(field);
478-
}
480+
addObjectAsDynamicMapping(field);
479481
} else {
480482
let fieldProps = getDefaultProperties(field);
481483

@@ -491,6 +493,12 @@ function _generateMappings(
491493
},
492494
isIndexModeTimeSeries
493495
);
496+
if (field.object_type) {
497+
// A group can have an object_type if it has been merged with an object during deduplication,
498+
// generate also the dynamic mapping for the object.
499+
addObjectAsDynamicMapping(field);
500+
mappings.hasDynamicTemplateMappings = true;
501+
}
494502
if (mappings.hasNonDynamicTemplateMappings) {
495503
fieldProps = {
496504
properties:

0 commit comments

Comments
 (0)