Skip to content

Commit b8317dc

Browse files
marshallmainkibanamachine
marshallmain
and
kibanamachine
committed
[Security Solution][Detections] Reduce detection engine reliance on _source (#89371)
* First pass at switching rules to depend on fields instead of _source * Fix tests * Change operator: excluded logic so missing fields are allowlisted Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
1 parent a7eaa71 commit b8317dc

20 files changed

Lines changed: 162 additions & 71 deletions

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,12 @@ export const sampleDocWithSortId = (
166166
ip: destIp ?? '127.0.0.1',
167167
},
168168
},
169+
fields: {
170+
someKey: ['someValue'],
171+
'@timestamp': ['2020-04-20T21:27:45+0000'],
172+
'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'],
173+
'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'],
174+
},
169175
sort: ['1234567891111'],
170176
});
171177

@@ -185,6 +191,11 @@ export const sampleDocNoSortId = (
185191
ip: ip ?? '127.0.0.1',
186192
},
187193
},
194+
fields: {
195+
someKey: ['someValue'],
196+
'@timestamp': ['2020-04-20T21:27:45+0000'],
197+
'source.ip': [ip ?? '127.0.0.1'],
198+
},
188199
sort: [],
189200
});
190201

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,12 @@ describe('create_signals', () => {
5656
],
5757
},
5858
},
59-
59+
fields: [
60+
{
61+
field: '*',
62+
include_unmapped: true,
63+
},
64+
],
6065
sort: [
6166
{
6267
'@timestamp': {
@@ -115,7 +120,12 @@ describe('create_signals', () => {
115120
],
116121
},
117122
},
118-
123+
fields: [
124+
{
125+
field: '*',
126+
include_unmapped: true,
127+
},
128+
],
119129
sort: [
120130
{
121131
'@timestamp': {
@@ -175,7 +185,12 @@ describe('create_signals', () => {
175185
],
176186
},
177187
},
178-
188+
fields: [
189+
{
190+
field: '*',
191+
include_unmapped: true,
192+
},
193+
],
179194
sort: [
180195
{
181196
'@timestamp': {
@@ -236,7 +251,12 @@ describe('create_signals', () => {
236251
],
237252
},
238253
},
239-
254+
fields: [
255+
{
256+
field: '*',
257+
include_unmapped: true,
258+
},
259+
],
240260
sort: [
241261
{
242262
'@timestamp': {
@@ -296,7 +316,12 @@ describe('create_signals', () => {
296316
],
297317
},
298318
},
299-
319+
fields: [
320+
{
321+
field: '*',
322+
include_unmapped: true,
323+
},
324+
],
300325
sort: [
301326
{
302327
'@timestamp': {
@@ -358,6 +383,12 @@ describe('create_signals', () => {
358383
],
359384
},
360385
},
386+
fields: [
387+
{
388+
field: '*',
389+
include_unmapped: true,
390+
},
391+
],
361392
aggregations: {
362393
tags: {
363394
terms: {

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ export const buildEventsSearchQuery = ({
8989
],
9090
},
9191
},
92+
fields: [
93+
{
94+
field: '*',
95+
include_unmapped: true,
96+
},
97+
],
9298
...(aggregations ? { aggregations } : {}),
9399
sort: [
94100
{

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ describe('transformThresholdResultsToEcs', () => {
6767
_id,
6868
_index: 'test',
6969
_source: {
70-
'@timestamp': '2020-04-20T21:27:45+0000',
70+
'@timestamp': ['2020-04-20T21:27:45+0000'],
7171
threshold_result: {
7272
count: 1,
7373
value: '127.0.0.1',

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ const getTransformedHits = (
7575
}
7676

7777
const source = {
78-
'@timestamp': get(timestampOverride ?? '@timestamp', hit._source),
78+
'@timestamp': get(timestampOverride ?? '@timestamp', hit.fields),
7979
threshold_result: {
8080
count: totalResults,
8181
value: ruleId,
@@ -104,10 +104,10 @@ const getTransformedHits = (
104104
}
105105

106106
const source = {
107-
'@timestamp': get(timestampOverride ?? '@timestamp', hit._source),
107+
'@timestamp': get(timestampOverride ?? '@timestamp', hit.fields),
108108
threshold_result: {
109109
count: docCount,
110-
value: get(threshold.field, hit._source),
110+
value: key,
111111
},
112112
};
113113

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_field_and_set_tuples.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ describe('filterEventsAgainstList', () => {
120120
exceptionItem,
121121
buildRuleMessage,
122122
});
123-
expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1')]);
123+
expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1'])]);
124124
});
125125

126126
test('it returns two matched sets as a JSON.stringify() set from the "events"', async () => {
@@ -133,7 +133,7 @@ describe('filterEventsAgainstList', () => {
133133
exceptionItem,
134134
buildRuleMessage,
135135
});
136-
expect([...matchedSet]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]);
136+
expect([...matchedSet]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]);
137137
});
138138

139139
test('it returns an array as a set as a JSON.stringify() array from the "events"', async () => {
@@ -282,7 +282,7 @@ describe('filterEventsAgainstList', () => {
282282
exceptionItem,
283283
buildRuleMessage,
284284
});
285-
expect([...matchedSet1]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]);
286-
expect([...matchedSet2]).toEqual([JSON.stringify('3.3.3.3'), JSON.stringify('5.5.5.5')]);
285+
expect([...matchedSet1]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]);
286+
expect([...matchedSet2]).toEqual([JSON.stringify(['3.3.3.3']), JSON.stringify(['5.5.5.5'])]);
287287
});
288288
});

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,9 @@ describe('createSetToFilterAgainst', () => {
6262
expect(listClient.searchListItemByValues).toHaveBeenCalledWith({
6363
listId: 'list-123',
6464
type: 'ip',
65-
value: ['1.1.1.1'],
65+
value: [['1.1.1.1']],
6666
});
67-
expect([...field]).toEqual([JSON.stringify('1.1.1.1')]);
67+
expect([...field]).toEqual([JSON.stringify(['1.1.1.1'])]);
6868
});
6969

7070
test('it returns 2 fields if the list returns 2 items', async () => {
@@ -81,9 +81,9 @@ describe('createSetToFilterAgainst', () => {
8181
expect(listClient.searchListItemByValues).toHaveBeenCalledWith({
8282
listId: 'list-123',
8383
type: 'ip',
84-
value: ['1.1.1.1', '2.2.2.2'],
84+
value: [['1.1.1.1'], ['2.2.2.2']],
8585
});
86-
expect([...field]).toEqual([JSON.stringify('1.1.1.1'), JSON.stringify('2.2.2.2')]);
86+
expect([...field]).toEqual([JSON.stringify(['1.1.1.1']), JSON.stringify(['2.2.2.2'])]);
8787
});
8888

8989
test('it returns 0 fields if the field does not match up to a valid field within the event', async () => {

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/create_set_to_filter_against.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
* 2.0.
66
*/
77

8-
import { get } from 'lodash/fp';
98
import { CreateSetToFilterAgainstOptions } from './types';
109

1110
/**
@@ -31,7 +30,7 @@ export const createSetToFilterAgainst = async <T>({
3130
buildRuleMessage,
3231
}: CreateSetToFilterAgainstOptions<T>): Promise<Set<unknown>> => {
3332
const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => {
34-
const valueField = get(field, searchResultItem._source);
33+
const valueField = searchResultItem.fields ? searchResultItem.fields[field] : undefined;
3534
if (valueField != null) {
3635
acc.add(valueField);
3736
}

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.test.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ describe('filterEvents', () => {
4040
{
4141
field: 'source.ip',
4242
operator: 'included',
43-
matchedSet: new Set([JSON.stringify('1.1.1.1')]),
43+
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
4444
},
4545
];
4646
const field = filterEvents({
@@ -56,7 +56,7 @@ describe('filterEvents', () => {
5656
{
5757
field: 'source.ip',
5858
operator: 'excluded',
59-
matchedSet: new Set([JSON.stringify('1.1.1.1')]),
59+
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
6060
},
6161
];
6262
const field = filterEvents({
@@ -72,7 +72,7 @@ describe('filterEvents', () => {
7272
{
7373
field: 'madeup.nonexistent', // field does not exist
7474
operator: 'included',
75-
matchedSet: new Set([JSON.stringify('1.1.1.1')]),
75+
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
7676
},
7777
];
7878
const field = filterEvents({
@@ -88,12 +88,12 @@ describe('filterEvents', () => {
8888
{
8989
field: 'source.ip',
9090
operator: 'included',
91-
matchedSet: new Set([JSON.stringify('1.1.1.1')]),
91+
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
9292
},
9393
{
9494
field: 'source.ip',
9595
operator: 'excluded',
96-
matchedSet: new Set([JSON.stringify('1.1.1.1')]),
96+
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
9797
},
9898
];
9999

‎x-pack/plugins/security_solution/server/lib/detection_engine/signals/filters/filter_events.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
* 2.0.
66
*/
77

8-
import { get } from 'lodash/fp';
98
import { SearchResponse } from '../../../types';
109
import { FilterEventsOptions } from './types';
1110

@@ -22,13 +21,17 @@ export const filterEvents = <T>({
2221
return events.filter((item) => {
2322
return fieldAndSetTuples
2423
.map((tuple) => {
25-
const eventItem = get(tuple.field, item._source);
26-
if (eventItem == null) {
27-
return true;
28-
} else if (tuple.operator === 'included') {
24+
const eventItem = item.fields ? item.fields[tuple.field] : undefined;
25+
if (tuple.operator === 'included') {
26+
if (eventItem == null) {
27+
return true;
28+
}
2929
// only create a signal if the event is not in the value list
3030
return !tuple.matchedSet.has(JSON.stringify(eventItem));
3131
} else if (tuple.operator === 'excluded') {
32+
if (eventItem == null) {
33+
return false;
34+
}
3235
// only create a signal if the event is in the value list
3336
return tuple.matchedSet.has(JSON.stringify(eventItem));
3437
} else {

0 commit comments

Comments
 (0)