Skip to content

Commit a7313cd

Browse files
legoguy1000andrewkroh
legoguy1000
and
andrewkroh
committed
[Filebeat] Update compatibility function to remove processor description on ES < 7.9.0 (#27774)
* Update Filebeat compatibility function to remove processor description field on ES < 7.9.0 * Fix function description Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
1 parent 9256741 commit a7313cd

3 files changed

Lines changed: 139 additions & 4 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
205205
- Auditd: Fix Top Exec Commands dashboard visualization. {pull}27638[27638]
206206
- Store offset in `log.offset` field of events from the filestream input. {pull}27688[27688]
207207
- Fix `httpjson` input rate limit processing and documentation. {pull}[]
208+
- Update Filebeat compatibility function to remove processor description field on ES < 7.9.0 {pull}27774[27774]
208209

209210
*Heartbeat*
210211

‎filebeat/fileset/compatibility.go‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,13 @@ var processorCompatibilityChecks = []processorCompatibility{
106106
},
107107
adaptConfig: deleteProcessor,
108108
},
109+
{
110+
procType: "*",
111+
checkVersion: func(esVersion *common.Version) bool {
112+
return esVersion.LessThan(common.MustNewVersion("7.9.0"))
113+
},
114+
adaptConfig: removeDescription,
115+
},
109116
}
110117

111118
// Processor represents and Ingest Node processor definition.
@@ -273,17 +280,17 @@ nextProcessor:
273280

274281
// Run compatibility checks on the processor.
275282
for _, proc := range processorCompatibilityChecks {
276-
if processor.Name() != proc.procType {
283+
if processor.Name() != proc.procType && proc.procType != "*" {
277284
continue
278285
}
279286

280287
if !proc.checkVersion(&esVersion) {
281288
continue
282289
}
283290

284-
processor, err = proc.adaptConfig(processor, log.With("processor_type", proc.procType, "processor_index", i))
291+
processor, err = proc.adaptConfig(processor, log.With("processor_type", processor.Name(), "processor_index", i))
285292
if err != nil {
286-
return fmt.Errorf("failed to adapt %q processor at index %d: %w", proc.procType, i, err)
293+
return fmt.Errorf("failed to adapt %q processor at index %d: %w", processor.Name(), i, err)
287294
}
288295
if processor.IsNil() {
289296
continue nextProcessor
@@ -408,3 +415,16 @@ func replaceConvertIP(processor Processor, log *logp.Logger) (Processor, error)
408415
log.Debug("processor output=", processor.String())
409416
return processor, nil
410417
}
418+
419+
// removeDescription removes the description config option so ES less than 7.9 will work.
420+
func removeDescription(processor Processor, log *logp.Logger) (Processor, error) {
421+
_, ok := processor.GetString("description")
422+
if !ok {
423+
return processor, nil
424+
}
425+
426+
log.Debug("Removing unsupported 'description' from processor.")
427+
processor.Delete("description")
428+
429+
return processor, nil
430+
}

‎filebeat/fileset/compatibility_test.go‎

Lines changed: 115 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,6 @@ func TestReplaceConvertIPWithGrok(t *testing.T) {
922922
"^%{IP:bar}$",
923923
},
924924
"ignore_missing": true,
925-
"description": "foo bar",
926925
"if": "condition",
927926
"ignore_failure": false,
928927
"tag": "myTag",
@@ -1341,3 +1340,118 @@ func TestReplaceAlternativeFlowProcessors(t *testing.T) {
13411340
})
13421341
}
13431342
}
1343+
1344+
func TestRemoveDescription(t *testing.T) {
1345+
cases := []struct {
1346+
name string
1347+
esVersion *common.Version
1348+
content map[string]interface{}
1349+
expected map[string]interface{}
1350+
isErrExpected bool
1351+
}{
1352+
{
1353+
name: "ES < 7.9.0",
1354+
esVersion: common.MustNewVersion("7.8.0"),
1355+
content: map[string]interface{}{
1356+
"processors": []interface{}{
1357+
map[string]interface{}{
1358+
"set": map[string]interface{}{
1359+
"field": "rule.name",
1360+
"value": "{{panw.panos.ruleset}}",
1361+
"description": "This is a description",
1362+
},
1363+
},
1364+
map[string]interface{}{
1365+
"script": map[string]interface{}{
1366+
"source": "abcd",
1367+
"lang": "painless",
1368+
"description": "This is a description",
1369+
},
1370+
},
1371+
}},
1372+
expected: map[string]interface{}{
1373+
"processors": []interface{}{
1374+
map[string]interface{}{
1375+
"set": map[string]interface{}{
1376+
"field": "rule.name",
1377+
"value": "{{panw.panos.ruleset}}",
1378+
},
1379+
},
1380+
map[string]interface{}{
1381+
"script": map[string]interface{}{
1382+
"source": "abcd",
1383+
"lang": "painless",
1384+
},
1385+
},
1386+
},
1387+
},
1388+
isErrExpected: false,
1389+
},
1390+
{
1391+
name: "ES == 7.9.0",
1392+
esVersion: common.MustNewVersion("7.9.0"),
1393+
content: map[string]interface{}{
1394+
"processors": []interface{}{
1395+
map[string]interface{}{
1396+
"set": map[string]interface{}{
1397+
"field": "rule.name",
1398+
"value": "{{panw.panos.ruleset}}",
1399+
"description": "This is a description",
1400+
},
1401+
},
1402+
}},
1403+
expected: map[string]interface{}{
1404+
"processors": []interface{}{
1405+
map[string]interface{}{
1406+
"set": map[string]interface{}{
1407+
"field": "rule.name",
1408+
"value": "{{panw.panos.ruleset}}",
1409+
"description": "This is a description",
1410+
},
1411+
},
1412+
},
1413+
},
1414+
isErrExpected: false,
1415+
},
1416+
{
1417+
name: "ES > 7.9.0",
1418+
esVersion: common.MustNewVersion("8.0.0"),
1419+
content: map[string]interface{}{
1420+
"processors": []interface{}{
1421+
map[string]interface{}{
1422+
"set": map[string]interface{}{
1423+
"field": "rule.name",
1424+
"value": "{{panw.panos.ruleset}}",
1425+
"description": "This is a description",
1426+
},
1427+
},
1428+
}},
1429+
expected: map[string]interface{}{
1430+
"processors": []interface{}{
1431+
map[string]interface{}{
1432+
"set": map[string]interface{}{
1433+
"field": "rule.name",
1434+
"value": "{{panw.panos.ruleset}}",
1435+
"description": "This is a description",
1436+
},
1437+
},
1438+
},
1439+
},
1440+
isErrExpected: false,
1441+
},
1442+
}
1443+
1444+
for _, test := range cases {
1445+
test := test
1446+
t.Run(test.name, func(t *testing.T) {
1447+
t.Parallel()
1448+
err := adaptPipelineForCompatibility(*test.esVersion, "foo-pipeline", test.content, logp.NewLogger(logName))
1449+
if test.isErrExpected {
1450+
assert.Error(t, err)
1451+
} else {
1452+
require.NoError(t, err)
1453+
assert.Equal(t, test.expected, test.content, test.name)
1454+
}
1455+
})
1456+
}
1457+
}

0 commit comments

Comments
 (0)