Skip to content

Commit e20d563

Browse files
mrodmmrodm
committed
Use just one schema to validate mappings not existing in preview
Make use of the External value in the ECS fields loaded into the schema to be able to distinguish which fields come from ECS and which ones come from the fields directory from the package. This allows to perform different tests when validating mappings
1 parent 0f980fa commit e20d563

3 files changed

Lines changed: 110 additions & 51 deletions

File tree

‎internal/fields/mappings.go‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,6 @@ type MappingValidator struct {
4040
indexTemplateName string
4141

4242
dataStreamName string
43-
44-
LocalSchema []FieldDefinition
4543
}
4644

4745
// MappingValidatorOption represents an optional flag that can be passed to CreateValidatorForMappings.
@@ -115,6 +113,14 @@ func WithMappingValidatorDataStream(dataStream string) MappingValidatorOption {
115113
}
116114
}
117115

116+
// WithMappingValidatorDataStream configures the Data Stream to query in Elasticsearch.
117+
func WithMappingValidatorFallbackSchema(schema []FieldDefinition) MappingValidatorOption {
118+
return func(v *MappingValidator) error {
119+
v.Schema = schema
120+
return nil
121+
}
122+
}
123+
118124
// CreateValidatorForMappings function creates a validator for the mappings.
119125
func CreateValidatorForMappings(fieldsParentDir string, esClient *elasticsearch.Client, opts ...MappingValidatorOption) (v *MappingValidator, err error) {
120126
p := packageRoot{}
@@ -130,6 +136,10 @@ func createValidatorForMappingsAndPackageRoot(fieldsParentDir string, finder pac
130136
}
131137
}
132138

139+
if len(v.Schema) > 0 {
140+
return v, nil
141+
}
142+
133143
fieldsDir := filepath.Join(fieldsParentDir, "fields")
134144

135145
var fdm *DependencyManager
@@ -151,10 +161,7 @@ func createValidatorForMappingsAndPackageRoot(fieldsParentDir string, finder pac
151161
return nil, fmt.Errorf("can't load fields from directory (path: %s): %w", fieldsDir, err)
152162
}
153163

154-
// Load field definitions from package in a different Validator variable
155-
// It allows to check whether or not a given mapping comes from ECS
156-
// Or just check if a packages contains some specific field definition (e.g. type array)
157-
v.LocalSchema = fields
164+
v.Schema = append(fields, v.Schema...)
158165
return v, nil
159166
}
160167

@@ -201,7 +208,7 @@ func (v *MappingValidator) ValidateIndexMappings(ctx context.Context) multierror
201208
// - If the same mapping exists in both, but they have different "type", there is some issue
202209
// - If there is a new mapping,
203210
// - It could come from a ECS definition, compare that mapping with the ECS field definitions
204-
// - Does this come from some dynamic template? ECS componente template or dynamic templates defined in the package? This mapping is valid
211+
// - Does this come from some dynamic template? ECS components template or dynamic templates defined in the package? This mapping is valid
205212
// - conditions found in current dynamic templates: match, path_match, path_unmatch, match_mapping_type, unmatch_mapping_type
206213
// - if it does not match, there should be some issue and it should be reported
207214
// - If the mapping is a constant_keyword type (e.g. data_stream.dataset), how to check the value?
@@ -224,7 +231,7 @@ func (v *MappingValidator) ValidateIndexMappings(ctx context.Context) multierror
224231
return errs.Unique()
225232
}
226233

227-
mappingErrs := compareMappings("", rawPreview, rawActual, v.Schema, v.LocalSchema)
234+
mappingErrs := compareMappings("", rawPreview, rawActual, v.Schema)
228235
errs = append(errs, mappingErrs...)
229236

230237
if len(errs) > 0 {
@@ -258,6 +265,9 @@ func isLocalFieldTypeArray(field string, schema []FieldDefinition) bool {
258265
if definition == nil {
259266
return false
260267
}
268+
if definition.External != "" {
269+
return false
270+
}
261271
return definition.Type == "array"
262272
}
263273

@@ -352,15 +362,22 @@ func isMultiFields(definition map[string]any) bool {
352362
return true
353363
}
354364

355-
func validateMappingInSchema(currentPath string, definition map[string]any, schema []FieldDefinition) error {
365+
func validateMappingInECSSchema(currentPath string, definition map[string]any, schema []FieldDefinition) error {
356366
found := FindElementDefinition(currentPath, schema)
357367
if found == nil {
358368
return fmt.Errorf("missing definition for path")
359369
}
360370

371+
if found.External != "ecs" {
372+
return fmt.Errorf("missing definition for path")
373+
}
374+
375+
logger.Debugf("Found ECS field %q for path %s (External %q)", found.Name, currentPath, found.External)
376+
361377
if found.Type != mappingParameter("type", definition) {
362378
return fmt.Errorf("mapping type does not match with ECS definition")
363379
}
380+
// any other field to validate here?
364381
return nil
365382
}
366383

@@ -450,7 +467,7 @@ func validateConstantKeywordField(path string, preview, actual map[string]any) (
450467
return isConstantKeyword, nil
451468
}
452469

453-
func compareMappings(path string, preview, actual map[string]any, ecsSchema, localSchema []FieldDefinition) multierror.Error {
470+
func compareMappings(path string, preview, actual map[string]any, schema []FieldDefinition) multierror.Error {
454471
var errs multierror.Error
455472

456473
isConstantKeywordType, err := validateConstantKeywordField(path, preview, actual)
@@ -484,7 +501,7 @@ func compareMappings(path string, preview, actual map[string]any, ecsSchema, loc
484501
errs = append(errs, fmt.Errorf("found invalid properties type in actual mappings for path %q: %w", path, err))
485502
}
486503
// logger.Debugf(">>> Comparing field with properties (object): %q", path)
487-
compareErrors := compareMappings(path, previewProperties, actualProperties, ecsSchema, localSchema)
504+
compareErrors := compareMappings(path, previewProperties, actualProperties, schema)
488505
errs = append(errs, compareErrors...)
489506

490507
if len(errs) == 0 {
@@ -508,21 +525,21 @@ func compareMappings(path string, preview, actual map[string]any, ecsSchema, loc
508525
errs = append(errs, fmt.Errorf("found invalid multi_fields type in actual mappings for path %q: %w", path, err))
509526
}
510527
// logger.Debugf(">>> Comparing multi_fields: %q", path)
511-
compareErrors := compareMappings(path, previewFields, actualFields, ecsSchema, localSchema)
528+
compareErrors := compareMappings(path, previewFields, actualFields, schema)
512529
errs = append(errs, compareErrors...)
513530
// not returning here to keep validating the other fields of this object if any
514531
}
515532

516533
// Compare and validate the elements under "properties": objects or fields and its parameters
517-
propertiesErrs := validateObjectProperties(path, containsMultifield, actual, preview, localSchema, ecsSchema)
534+
propertiesErrs := validateObjectProperties(path, containsMultifield, actual, preview, schema)
518535
errs = append(errs, propertiesErrs...)
519536
if len(errs) == 0 {
520537
return nil
521538
}
522539
return errs.Unique()
523540
}
524541

525-
func validateObjectProperties(path string, containsMultifield bool, actual, preview map[string]any, localSchema, ecsSchema []FieldDefinition) multierror.Error {
542+
func validateObjectProperties(path string, containsMultifield bool, actual, preview map[string]any, schema []FieldDefinition) multierror.Error {
526543
var errs multierror.Error
527544
for key, value := range actual {
528545
if containsMultifield && key == "fields" {
@@ -544,14 +561,14 @@ func validateObjectProperties(path string, containsMultifield bool, actual, prev
544561
continue
545562
}
546563
errs = append(errs,
547-
validateMappingsNotInPreview(currentPath, childField, localSchema, ecsSchema)...,
564+
validateMappingsNotInPreview(currentPath, childField, schema)...,
548565
)
549566
}
550567

551568
continue
552569
}
553570

554-
fieldErrs := validateObjectMappingAndParameters(preview[key], value, currentPath, ecsSchema, localSchema)
571+
fieldErrs := validateObjectMappingAndParameters(preview[key], value, currentPath, schema)
555572
errs = append(errs, fieldErrs...)
556573
}
557574
if len(errs) == 0 {
@@ -562,7 +579,7 @@ func validateObjectProperties(path string, containsMultifield bool, actual, prev
562579

563580
// validateMappingsNotInPreview validates the object and the nested objects in the current path with other resources
564581
// like ECS schema, dynamic templates or local fields defined in the package (type array).
565-
func validateMappingsNotInPreview(currentPath string, childField map[string]any, localSchema, ecsSchema []FieldDefinition) multierror.Error {
582+
func validateMappingsNotInPreview(currentPath string, childField map[string]any, schema []FieldDefinition) multierror.Error {
566583
var errs multierror.Error
567584
logger.Debugf("Calculating flatten fields for %s", currentPath)
568585
flattenFields, err := flattenMappings(currentPath, childField)
@@ -580,7 +597,7 @@ func validateMappingsNotInPreview(currentPath string, childField map[string]any,
580597
continue
581598
}
582599

583-
if isLocalFieldTypeArray(fieldPath, localSchema) {
600+
if isLocalFieldTypeArray(fieldPath, schema) {
584601
logger.Debugf("Found field definition with type array, skipping path: %q", fieldPath)
585602
continue
586603
}
@@ -589,7 +606,7 @@ func validateMappingsNotInPreview(currentPath string, childField map[string]any,
589606
// just raise an error if both validation processes fail
590607

591608
// are all fields under this key defined in ECS?
592-
err = validateMappingInSchema(fieldPath, def, ecsSchema)
609+
err = validateMappingInECSSchema(fieldPath, def, schema)
593610
if err != nil {
594611
logger.Warnf("undefined path %q (pending to check dynamic templates)", fieldPath)
595612
errs = append(errs, fmt.Errorf("field %q is undefined: %w", fieldPath, err))
@@ -600,7 +617,7 @@ func validateMappingsNotInPreview(currentPath string, childField map[string]any,
600617

601618
// validateObjectMappingAndParameters validates the current object or field parameter (currentPath) comparing the values
602619
// in the actual mapping with the values in the preview mapping.
603-
func validateObjectMappingAndParameters(previewValue, actualValue any, currentPath string, ecsSchema, localSchema []FieldDefinition) multierror.Error {
620+
func validateObjectMappingAndParameters(previewValue, actualValue any, currentPath string, schema []FieldDefinition) multierror.Error {
604621
var errs multierror.Error
605622
switch actualValue.(type) {
606623
case map[string]any:
@@ -614,7 +631,7 @@ func validateObjectMappingAndParameters(previewValue, actualValue any, currentPa
614631
errs = append(errs, fmt.Errorf("unexpected type in actual mappings for path: %q", currentPath))
615632
}
616633
logger.Debugf(">>>> Comparing Mappings map[string]any: path %s", currentPath)
617-
errs = append(errs, compareMappings(currentPath, previewField, actualField, ecsSchema, localSchema)...)
634+
errs = append(errs, compareMappings(currentPath, previewField, actualField, schema)...)
618635
case any:
619636
// Validate each setting/parameter of the mapping
620637
// If a mapping exist in both preview and actual, they should be the same. But forcing to compare each parameter just in case
@@ -623,15 +640,16 @@ func validateObjectMappingAndParameters(previewValue, actualValue any, currentPa
623640
if previewValue == actualValue {
624641
return nil
625642
}
643+
// Get the string representation via JSON Marshalling
626644
previewData, err := json.Marshal(previewValue)
627645
if err != nil {
628-
errs = append(errs, fmt.Errorf("error unmarshalling preview value %s (path: %s): %w", previewValue, currentPath, err))
646+
errs = append(errs, fmt.Errorf("error marshalling preview value %s (path: %s): %w", previewValue, currentPath, err))
629647
return errs
630648
}
631649

632650
actualData, err := json.Marshal(actualValue)
633651
if err != nil {
634-
errs = append(errs, fmt.Errorf("error unmarshalling actual value %s (path: %s): %w", actualValue, currentPath, err))
652+
errs = append(errs, fmt.Errorf("error marshalling actual value %s (path: %s): %w", actualValue, currentPath, err))
635653
return errs
636654
}
637655
errs = append(errs, fmt.Errorf("unexpected value found in mapping for field %q: preview mappings value (%s) different from the actual mappings value (%s)", currentPath, string(previewData), string(actualData)))

0 commit comments

Comments
 (0)