Skip to content

Commit 8395704

Browse files
rylndrylnd
committed
[Detection Engine] Unskip some flaky tests, add better failure messages (#230318)
## Summary While the tests affected by this PR are varied, the changes contained here fall under one of two categories: 1. Unskipping flaky tests 2. Adding better test assertions (in order to produce more actionable failures later) ### Related Issues * Closes #224699 * Closes #224780 * Closes #220943 * Closes #202940 * Closes #202945 ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios (cherry picked from commit 0a3e7bb) # Conflicts: # x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql.ts # x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts # x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql.ts
1 parent 6a93f5b commit 8395704

9 files changed

Lines changed: 223 additions & 226 deletions

File tree

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/exceptions/workflows/basic_license_essentials_tier/rule_exceptions_execution.ts‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ export default ({ getService }: FtrProviderContext) => {
122122
await waitForRuleSuccess({ supertest, log, id: createdId });
123123
await waitForAlertsToBePresent(supertest, log, 10, [createdId]);
124124
const alertsOpen = await getAlertsByIds(supertest, log, [createdId]);
125-
expect(alertsOpen.hits.hits.length).toEqual(10);
125+
expect(alertsOpen.hits.hits).toHaveLength(10);
126126
});
127127

128128
it('should be able to execute against an exception list that does include valid entries and get back 0 alerts', async () => {
@@ -149,7 +149,7 @@ export default ({ getService }: FtrProviderContext) => {
149149
],
150150
]);
151151
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
152-
expect(alertsOpen.hits.hits.length).toEqual(0);
152+
expect(alertsOpen.hits.hits).toHaveLength(0);
153153
});
154154

155155
it('should be able to execute against an exception list that does include valid case sensitive entries and get back 0 alerts', async () => {
@@ -201,10 +201,10 @@ export default ({ getService }: FtrProviderContext) => {
201201
const alertsOpen2 = await getOpenAlerts(supertest, log, es, createdRule2);
202202
// Expect alerts here because all values are "Ubuntu"
203203
// and exception is one of ["ubuntu"]
204-
expect(alertsOpen.hits.hits.length).toEqual(10);
204+
expect(alertsOpen.hits.hits).toHaveLength(10);
205205
// Expect no alerts here because all values are "Ubuntu"
206206
// and exception is one of ["ubuntu", "Ubuntu"]
207-
expect(alertsOpen2.hits.hits.length).toEqual(0);
207+
expect(alertsOpen2.hits.hits).toHaveLength(0);
208208
});
209209

210210
it('generates no alerts when an exception is added for an EQL rule', async () => {
@@ -223,7 +223,7 @@ export default ({ getService }: FtrProviderContext) => {
223223
],
224224
]);
225225
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
226-
expect(alertsOpen.hits.hits.length).toEqual(0);
226+
expect(alertsOpen.hits.hits).toHaveLength(0);
227227
});
228228

229229
it('generates no alerts when an exception is added for a threshold rule', async () => {
@@ -245,7 +245,7 @@ export default ({ getService }: FtrProviderContext) => {
245245
],
246246
]);
247247
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
248-
expect(alertsOpen.hits.hits.length).toEqual(0);
248+
expect(alertsOpen.hits.hits).toHaveLength(0);
249249
});
250250

251251
it('generates no alerts when an exception is added for a threat match rule', async () => {
@@ -288,8 +288,9 @@ export default ({ getService }: FtrProviderContext) => {
288288
],
289289
]);
290290
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
291-
expect(alertsOpen.hits.hits.length).toEqual(0);
291+
expect(alertsOpen.hits.hits).toHaveLength(0);
292292
});
293+
293294
describe('rules with value list exceptions', () => {
294295
beforeEach(async () => {
295296
await createListsIndex(supertest, log);
@@ -328,7 +329,7 @@ export default ({ getService }: FtrProviderContext) => {
328329
],
329330
]);
330331
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
331-
expect(alertsOpen.hits.hits.length).toEqual(0);
332+
expect(alertsOpen.hits.hits).toHaveLength(0);
332333
});
333334

334335
it('generates no alerts when a value list exception is added for a threat match rule', async () => {
@@ -376,7 +377,7 @@ export default ({ getService }: FtrProviderContext) => {
376377
],
377378
]);
378379
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
379-
expect(alertsOpen.hits.hits.length).toEqual(0);
380+
expect(alertsOpen.hits.hits).toHaveLength(0);
380381
});
381382

382383
it('generates no alerts when a value list exception is added for a threshold rule', async () => {
@@ -413,7 +414,7 @@ export default ({ getService }: FtrProviderContext) => {
413414
],
414415
]);
415416
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
416-
expect(alertsOpen.hits.hits.length).toEqual(0);
417+
expect(alertsOpen.hits.hits).toHaveLength(0);
417418
});
418419

419420
it('generates no alerts when a value list exception is added for an EQL rule', async () => {
@@ -438,8 +439,9 @@ export default ({ getService }: FtrProviderContext) => {
438439
],
439440
]);
440441
const alertsOpen = await getOpenAlerts(supertest, log, es, createdRule);
441-
expect(alertsOpen.hits.hits.length).toEqual(0);
442+
expect(alertsOpen.hits.hits).toHaveLength(0);
442443
});
444+
443445
it('should Not allow deleting value list when there are references and ignoreReferences is false', async () => {
444446
const valueListId = 'value-list-id.txt';
445447
await importFile(supertest, log, 'keyword', ['suricata-sensor-amsterdam'], valueListId);

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql.ts‎

Lines changed: 29 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,7 @@ export default ({ getService }: FtrProviderContext) => {
8282
const auditPath = dataPathBuilder.getPath('auditbeat/hosts');
8383
const packetBeatPath = dataPathBuilder.getPath('packetbeat/default');
8484

85-
// Failing: See https://github.com/elastic/kibana/issues/209024
86-
describe.skip('@ess @serverless @serverlessQA EQL type rules', () => {
85+
describe('@ess @serverless @serverlessQA EQL type rules', () => {
8786
const { indexListOfDocuments } = dataGeneratorFactory({
8887
es,
8988
index: 'ecs_compliant',
@@ -116,7 +115,7 @@ export default ({ getService }: FtrProviderContext) => {
116115
};
117116
const createdRule = await createRule(supertest, log, rule);
118117
const alerts = await getAlerts(supertest, log, es, createdRule);
119-
expect(alerts.hits.hits.length).eql(1);
118+
expect(alerts.hits.hits).toHaveLength(1);
120119
const fullAlert = alerts.hits.hits[0]._source;
121120
if (!fullAlert) {
122121
return expect(fullAlert).to.be.ok();
@@ -285,7 +284,7 @@ export default ({ getService }: FtrProviderContext) => {
285284
};
286285
const { previewId } = await previewRule({ supertest, rule });
287286
const previewAlerts = await getPreviewAlerts({ es, previewId, size: maxAlerts * 2 });
288-
expect(previewAlerts.length).eql(maxAlerts);
287+
expect(previewAlerts).toHaveLength(maxAlerts);
289288
});
290289

291290
it('generates max alerts warning when circuit breaker is hit', async () => {
@@ -304,7 +303,7 @@ export default ({ getService }: FtrProviderContext) => {
304303
};
305304
const { previewId } = await previewRule({ supertest, rule });
306305
const previewAlerts = await getPreviewAlerts({ es, previewId });
307-
expect(previewAlerts.length).eql(1);
306+
expect(previewAlerts).toHaveLength(1);
308307
const fullAlert = previewAlerts[0]._source;
309308
if (!fullAlert) {
310309
return expect(fullAlert).to.be.ok();
@@ -374,7 +373,7 @@ export default ({ getService }: FtrProviderContext) => {
374373
};
375374
const { previewId } = await previewRule({ supertest, rule });
376375
const previewAlerts = await getPreviewAlerts({ es, previewId });
377-
expect(previewAlerts.length).eql(3);
376+
expect(previewAlerts).toHaveLength(3);
378377

379378
const createdAtHits = previewAlerts.map((hit) => hit._source?.created_at).sort();
380379
expect(createdAtHits).to.eql([1622676785, 1622676790, 1622676795]);
@@ -388,7 +387,7 @@ export default ({ getService }: FtrProviderContext) => {
388387
};
389388
const { previewId } = await previewRule({ supertest, rule });
390389
const previewAlerts = await getPreviewAlerts({ es, previewId });
391-
expect(previewAlerts.length).eql(3);
390+
expect(previewAlerts).toHaveLength(3);
392391

393392
const createdAtHits = previewAlerts.map((hit) => hit._source?.locale);
394393
expect(createdAtHits).to.eql(['es', 'pt', 'ua']);
@@ -668,7 +667,7 @@ export default ({ getService }: FtrProviderContext) => {
668667

669668
const previewAlerts = await getPreviewAlerts({ es, previewId, sort: ['agent.name'] });
670669

671-
expect(previewAlerts).to.have.length(3);
670+
expect(previewAlerts).toHaveLength(3);
672671

673672
const buildingBlockAlerts = previewAlerts.filter(
674673
(alert) => alert._source?.['kibana.alert.building_block_type']
@@ -712,11 +711,11 @@ export default ({ getService }: FtrProviderContext) => {
712711
// For EQL rules, max_alerts is the maximum number of detected sequences: each sequence has a building block
713712
// alert for each event in the sequence, so max_alerts=200 results in 400 building blocks in addition to
714713
// 200 regular alerts
715-
expect(previewAlerts.length).eql(maxAlerts * 3);
714+
expect(previewAlerts).toHaveLength(maxAlerts * 3);
716715
const shellAlerts = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 2);
717716
const buildingBlocks = previewAlerts.filter((alert) => alert._source?.[ALERT_DEPTH] === 1);
718-
expect(shellAlerts.length).eql(maxAlerts);
719-
expect(buildingBlocks.length).eql(maxAlerts * 2);
717+
expect(shellAlerts).toHaveLength(maxAlerts);
718+
expect(buildingBlocks).toHaveLength(maxAlerts * 2);
720719
});
721720

722721
it('generates alerts when an index name contains special characters to encode', async () => {
@@ -726,7 +725,7 @@ export default ({ getService }: FtrProviderContext) => {
726725
};
727726
const { previewId } = await previewRule({ supertest, rule });
728727
const previewAlerts = await getPreviewAlerts({ es, previewId });
729-
expect(previewAlerts.length).eql(1);
728+
expect(previewAlerts).toHaveLength(1);
730729
});
731730

732731
it('uses the provided filters', async () => {
@@ -772,7 +771,7 @@ export default ({ getService }: FtrProviderContext) => {
772771
};
773772
const { previewId } = await previewRule({ supertest, rule });
774773
const previewAlerts = await getPreviewAlerts({ es, previewId });
775-
expect(previewAlerts.length).eql(2);
774+
expect(previewAlerts).toHaveLength(2);
776775
});
777776

778777
describe('with host risk index', () => {
@@ -791,7 +790,7 @@ export default ({ getService }: FtrProviderContext) => {
791790
};
792791
const { previewId } = await previewRule({ supertest, rule });
793792
const previewAlerts = await getPreviewAlerts({ es, previewId });
794-
expect(previewAlerts.length).eql(1);
793+
expect(previewAlerts).toHaveLength(1);
795794
const fullAlert = previewAlerts[0]._source;
796795
if (!fullAlert) {
797796
return expect(fullAlert).to.be.ok();
@@ -842,7 +841,7 @@ export default ({ getService }: FtrProviderContext) => {
842841
expect(_log.warnings).to.eql([expectedWarning]);
843842

844843
const previewAlerts = await getPreviewAlerts({ es, previewId });
845-
expect(previewAlerts.length).to.be.greaterThan(0);
844+
expect(previewAlerts).not.toHaveLength(0);
846845
});
847846

848847
it('specifying only timestamp_override results in alert creation with an expected warning', async () => {
@@ -860,7 +859,7 @@ export default ({ getService }: FtrProviderContext) => {
860859
expect(_log.warnings).to.eql([expectedWarning]);
861860

862861
const previewAlerts = await getPreviewAlerts({ es, previewId });
863-
expect(previewAlerts.length).to.be.greaterThan(0);
862+
expect(previewAlerts).not.toHaveLength(0);
864863
});
865864

866865
it('specifying both timestamp_override and timestamp_field results in alert creation with an expected warning', async () => {
@@ -879,7 +878,7 @@ export default ({ getService }: FtrProviderContext) => {
879878
expect(_log.warnings).to.eql([expectedWarning]);
880879

881880
const previewAlerts = await getPreviewAlerts({ es, previewId });
882-
expect(previewAlerts.length).to.be.greaterThan(0);
881+
expect(previewAlerts).not.toHaveLength(0);
883882
});
884883
});
885884

@@ -951,7 +950,7 @@ export default ({ getService }: FtrProviderContext) => {
951950
expect(_log.warnings).to.be.empty();
952951
const previewAlerts = await getPreviewAlerts({ es, previewId });
953952

954-
expect(previewAlerts).to.have.length(3);
953+
expect(previewAlerts).toHaveLength(3);
955954
});
956955
});
957956

@@ -1024,7 +1023,7 @@ export default ({ getService }: FtrProviderContext) => {
10241023

10251024
const createdRule = await createRule(supertest, log, rule);
10261025
const alerts = await getAlerts(supertest, log, es, createdRule);
1027-
expect(alerts.hits.hits.length).equal(3);
1026+
expect(alerts.hits.hits).toHaveLength(3);
10281027
expect(alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('scheduled');
10291028

10301029
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
@@ -1034,7 +1033,7 @@ export default ({ getService }: FtrProviderContext) => {
10341033

10351034
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
10361035
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1037-
expect(allNewAlerts.hits.hits.length).equal(6);
1036+
expect(allNewAlerts.hits.hits).toHaveLength(6);
10381037
expect(allNewAlerts.hits.hits[5]?._source?.[ALERT_RULE_EXECUTION_TYPE]).equal('manual');
10391038

10401039
const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], {
@@ -1044,7 +1043,7 @@ export default ({ getService }: FtrProviderContext) => {
10441043

10451044
await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
10461045
const allNewAlertsAfter2ManualRuns = await getAlerts(supertest, log, es, createdRule);
1047-
expect(allNewAlertsAfter2ManualRuns.hits.hits.length).equal(6);
1046+
expect(allNewAlertsAfter2ManualRuns.hits.hits).toHaveLength(6);
10481047
});
10491048

10501049
it('does not alert if the manual run overlaps with a previous scheduled rule execution', async () => {
@@ -1083,7 +1082,7 @@ export default ({ getService }: FtrProviderContext) => {
10831082
const createdRule = await createRule(supertest, log, rule);
10841083
const alerts = await getAlerts(supertest, log, es, createdRule);
10851084

1086-
expect(alerts.hits.hits.length).equal(3);
1085+
expect(alerts.hits.hits).toHaveLength(3);
10871086

10881087
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
10891088
startDate: moment(firstTimestamp).subtract(5, 'm'),
@@ -1092,7 +1091,7 @@ export default ({ getService }: FtrProviderContext) => {
10921091

10931092
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
10941093
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1095-
expect(allNewAlerts.hits.hits.length).equal(3);
1094+
expect(allNewAlerts.hits.hits).toHaveLength(3);
10961095
});
10971096

10981097
it('supression per rule execution should work for manual rule runs', async () => {
@@ -1136,7 +1135,7 @@ export default ({ getService }: FtrProviderContext) => {
11361135
const createdRule = await createRule(supertest, log, rule);
11371136
const alerts = await getAlerts(supertest, log, es, createdRule);
11381137

1139-
expect(alerts.hits.hits.length).equal(0);
1138+
expect(alerts.hits.hits).toHaveLength(0);
11401139

11411140
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
11421141
startDate: moment(firstTimestamp).subtract(5, 'm'),
@@ -1145,7 +1144,7 @@ export default ({ getService }: FtrProviderContext) => {
11451144

11461145
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
11471146
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1148-
expect(allNewAlerts.hits.hits.length).equal(1);
1147+
expect(allNewAlerts.hits.hits).toHaveLength(1);
11491148

11501149
expect(allNewAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).equal(2);
11511150
});
@@ -1181,7 +1180,7 @@ export default ({ getService }: FtrProviderContext) => {
11811180
const createdRule = await createRule(supertest, log, rule);
11821181
const alerts = await getAlerts(supertest, log, es, createdRule);
11831182

1184-
expect(alerts.hits.hits.length).equal(0);
1183+
expect(alerts.hits.hits).toHaveLength(0);
11851184

11861185
// generate alert in the past
11871186
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
@@ -1190,7 +1189,7 @@ export default ({ getService }: FtrProviderContext) => {
11901189
});
11911190
await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
11921191
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
1193-
expect(allNewAlerts.hits.hits.length).equal(1);
1192+
expect(allNewAlerts.hits.hits).toHaveLength(1);
11941193

11951194
// now we will ingest new event, and manual rule run should update original alert
11961195
const secondDocument = {
@@ -1210,9 +1209,9 @@ export default ({ getService }: FtrProviderContext) => {
12101209

12111210
await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
12121211
const updatedAlerts = await getAlerts(supertest, log, es, createdRule);
1213-
expect(updatedAlerts.hits.hits.length).equal(1);
1212+
expect(updatedAlerts.hits.hits).toHaveLength(1);
12141213

1215-
expect(updatedAlerts.hits.hits.length).equal(1);
1214+
expect(updatedAlerts.hits.hits).toHaveLength(1);
12161215

12171216
expect(updatedAlerts.hits.hits[0]._source?.[ALERT_SUPPRESSION_DOCS_COUNT]).equal(1);
12181217
});
@@ -1236,7 +1235,7 @@ export default ({ getService }: FtrProviderContext) => {
12361235

12371236
const requests = logs[0].requests;
12381237

1239-
expect(requests).to.have.length(1);
1238+
expect(requests).toHaveLength(1);
12401239
expect(requests![0].description).to.be('EQL request to find all matches');
12411240
expect(requests![0].request).to.contain(
12421241
'POST /auditbeat-*/_eql/search?allow_no_indices=true'

0 commit comments

Comments
 (0)