Skip to content

Commit 8acfb0e

Browse files
darnautovkibanamachine
darnautov
authored and
kibanamachine
committed
[ML] Remove script fields from the Anomaly detection alerting rule executor (#101607)
* [ML] remove script fields * [ML] fix initial score
1 parent 9838db3 commit 8acfb0e

1 file changed

Lines changed: 39 additions & 99 deletions

File tree

‎x-pack/plugins/ml/server/lib/alerts/alerting_service.ts‎

Lines changed: 39 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ type AggResultsResponse = { key?: number } & {
4141
};
4242
};
4343

44+
const TIME_RANGE_PADDING = 10;
45+
4446
/**
4547
* Mapping for result types and corresponding score fields.
4648
*/
@@ -63,43 +65,6 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
6365
};
6466
};
6567

66-
const getCommonScriptedFields = () => {
67-
return {
68-
start: {
69-
script: {
70-
lang: 'painless',
71-
source: `LocalDateTime.ofEpochSecond((doc["timestamp"].value.getMillis()-((doc["bucket_span"].value * 1000)
72-
* params.padding)) / 1000, 0, ZoneOffset.UTC).toString()+\":00.000Z\"`,
73-
params: {
74-
padding: 10,
75-
},
76-
},
77-
},
78-
end: {
79-
script: {
80-
lang: 'painless',
81-
source: `LocalDateTime.ofEpochSecond((doc["timestamp"].value.getMillis()+((doc["bucket_span"].value * 1000)
82-
* params.padding)) / 1000, 0, ZoneOffset.UTC).toString()+\":00.000Z\"`,
83-
params: {
84-
padding: 10,
85-
},
86-
},
87-
},
88-
timestamp_epoch: {
89-
script: {
90-
lang: 'painless',
91-
source: 'doc["timestamp"].value.getMillis()/1000',
92-
},
93-
},
94-
timestamp_iso8601: {
95-
script: {
96-
lang: 'painless',
97-
source: 'doc["timestamp"].value',
98-
},
99-
},
100-
};
101-
};
102-
10368
/**
10469
* Builds an agg query based on the requested result type.
10570
* @param resultType
@@ -110,9 +75,9 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
11075
severity: number,
11176
useInitialScore?: boolean
11277
) => {
113-
const influencerScoreField = `${useInitialScore ? 'initial_' : ''}influencer_score`;
114-
const recordScoreField = `${useInitialScore ? 'initial_' : ''}record_score`;
115-
const bucketScoreField = `${useInitialScore ? 'initial_' : ''}anomaly_score`;
78+
const influencerScoreField = getScoreFields(ANOMALY_RESULT_TYPE.INFLUENCER, useInitialScore);
79+
const recordScoreField = getScoreFields(ANOMALY_RESULT_TYPE.RECORD, useInitialScore);
80+
const bucketScoreField = getScoreFields(ANOMALY_RESULT_TYPE.BUCKET, useInitialScore);
11681

11782
return {
11883
influencer_results: {
@@ -140,27 +105,13 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
140105
'influencer_field_name',
141106
'influencer_field_value',
142107
'influencer_score',
108+
'initial_influencer_score',
143109
'is_interim',
144110
'job_id',
111+
'bucket_span',
145112
],
146113
},
147114
size: 3,
148-
script_fields: {
149-
...getCommonScriptedFields(),
150-
score: {
151-
script: {
152-
lang: 'painless',
153-
source: `Math.floor(doc["${influencerScoreField}"].value)`,
154-
},
155-
},
156-
unique_key: {
157-
script: {
158-
lang: 'painless',
159-
source:
160-
'doc["timestamp"].value + "_" + doc["influencer_field_name"].value + "_" + doc["influencer_field_value"].value',
161-
},
162-
},
163-
},
164115
},
165116
},
166117
},
@@ -188,6 +139,7 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
188139
'result_type',
189140
'timestamp',
190141
'record_score',
142+
'initial_record_score',
191143
'is_interim',
192144
'function',
193145
'field_name',
@@ -199,24 +151,10 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
199151
'partition_field_value',
200152
'job_id',
201153
'detector_index',
154+
'bucket_span',
202155
],
203156
},
204157
size: 3,
205-
script_fields: {
206-
...getCommonScriptedFields(),
207-
score: {
208-
script: {
209-
lang: 'painless',
210-
source: `Math.floor(doc["${recordScoreField}"].value)`,
211-
},
212-
},
213-
unique_key: {
214-
script: {
215-
lang: 'painless',
216-
source: 'doc["timestamp"].value + "_" + doc["function"].value',
217-
},
218-
},
219-
},
220158
},
221159
},
222160
},
@@ -247,25 +185,12 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
247185
'result_type',
248186
'timestamp',
249187
'anomaly_score',
188+
'initial_anomaly_score',
250189
'is_interim',
190+
'bucket_span',
251191
],
252192
},
253193
size: 1,
254-
script_fields: {
255-
...getCommonScriptedFields(),
256-
score: {
257-
script: {
258-
lang: 'painless',
259-
source: `Math.floor(doc["${bucketScoreField}"].value)`,
260-
},
261-
},
262-
unique_key: {
263-
script: {
264-
lang: 'painless',
265-
source: 'doc["timestamp"].value',
266-
},
267-
},
268-
},
269194
},
270195
},
271196
},
@@ -282,6 +207,10 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
282207
return source.job_id;
283208
};
284209

210+
const getScoreFields = (resultType: AnomalyResultType, useInitialScore?: boolean) => {
211+
return `${useInitialScore ? 'initial_' : ''}${resultTypeScoreMapping[resultType]}`;
212+
};
213+
285214
const getRecordKey = (source: AnomalyRecordDoc): string => {
286215
let alertInstanceKey = `${source.job_id}_${source.timestamp}`;
287216

@@ -294,18 +223,23 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
294223
return alertInstanceKey;
295224
};
296225

297-
const getResultsFormatter = (resultType: AnomalyResultType) => {
226+
/**
227+
* Returns a callback for formatting elasticsearch aggregation response
228+
* to the alert context.
229+
* @param resultType
230+
*/
231+
const getResultsFormatter = (resultType: AnomalyResultType, useInitialScore: boolean = false) => {
298232
const resultsLabel = getAggResultsLabel(resultType);
299233
return (v: AggResultsResponse): AlertExecutionResult | undefined => {
300234
const aggTypeResults = v[resultsLabel.aggGroupLabel];
301235
if (aggTypeResults.doc_count === 0) {
302236
return;
303237
}
304-
305238
const requestedAnomalies = aggTypeResults[resultsLabel.topHitsLabel].hits.hits;
306-
307239
const topAnomaly = requestedAnomalies[0];
308240
const alertInstanceKey = getAlertInstanceKey(topAnomaly._source);
241+
const timestamp = topAnomaly._source.timestamp;
242+
const bucketSpanInSeconds = topAnomaly._source.bucket_span;
309243

310244
return {
311245
count: aggTypeResults.doc_count,
@@ -315,26 +249,32 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
315249
alertInstanceKey,
316250
jobIds: [...new Set(requestedAnomalies.map((h) => h._source.job_id))],
317251
isInterim: requestedAnomalies.some((h) => h._source.is_interim),
318-
timestamp: topAnomaly._source.timestamp,
319-
timestampIso8601: topAnomaly.fields.timestamp_iso8601[0],
320-
timestampEpoch: topAnomaly.fields.timestamp_epoch[0],
321-
score: topAnomaly.fields.score[0],
252+
timestamp,
253+
timestampIso8601: new Date(timestamp).toISOString(),
254+
timestampEpoch: timestamp / 1000,
255+
score: Math.floor(topAnomaly._source[getScoreFields(resultType, useInitialScore)]),
322256
bucketRange: {
323-
start: topAnomaly.fields.start[0],
324-
end: topAnomaly.fields.end[0],
257+
start: new Date(
258+
timestamp - bucketSpanInSeconds * 1000 * TIME_RANGE_PADDING
259+
).toISOString(),
260+
end: new Date(timestamp + bucketSpanInSeconds * 1000 * TIME_RANGE_PADDING).toISOString(),
325261
},
326262
topRecords: v.record_results.top_record_hits.hits.hits.map((h) => {
327263
return {
328264
...h._source,
329-
score: h.fields.score[0],
265+
score: Math.floor(
266+
h._source[getScoreFields(ANOMALY_RESULT_TYPE.RECORD, useInitialScore)]
267+
),
330268
unique_key: getRecordKey(h._source),
331269
};
332270
}) as RecordAnomalyAlertDoc[],
333271
topInfluencers: v.influencer_results.top_influencer_hits.hits.hits.map((h) => {
334272
return {
335273
...h._source,
336-
score: h.fields.score[0],
337-
unique_key: h.fields.unique_key[0],
274+
score: Math.floor(
275+
h._source[getScoreFields(ANOMALY_RESULT_TYPE.INFLUENCER, useInitialScore)]
276+
),
277+
unique_key: `${h._source.timestamp}_${h._source.influencer_field_name}_${h._source.influencer_field_value}`,
338278
};
339279
}) as InfluencerAnomalyAlertDoc[],
340280
};
@@ -447,7 +387,7 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da
447387

448388
const resultsLabel = getAggResultsLabel(params.resultType);
449389

450-
const formatter = getResultsFormatter(params.resultType);
390+
const formatter = getResultsFormatter(params.resultType, !!previewTimeInterval);
451391

452392
return (previewTimeInterval
453393
? (result as {

0 commit comments

Comments
 (0)