Skip to content

Commit 76908c5

Browse files
maximpnmaximpn
authored
[Security Solution] Fix real prebuilt rules package related flakiness in integration tests (elastic#227689)
**Resolves: elastic#220333 **Resolves: elastic#227571 **Resolves: elastic#219407 ## Summary This PR fixes flakiness caused by the real prebuilt rules package installation. ## Details Multiple Rule Management API endpoints install the prebuilt rules package under the hood when it's not installed. In normal case this installation happens from the EPR. The problem is that EPR might be unavailable for some period of time or there network issues blocking EPR access. On top of that real package is heavy and requires some time and resources to be installed. While the major part of our functionality doesn't depend on the real prebuilt rules package and the concrete rules there. On top of that API endpoints depending on the real prebuilt rules package fail with HTTP 500 error when EPR is unavailable. This PR extends the Functional Test Runner with `rootHooks.beforeAll` configuration option which is used to install a lightweight mock prebuilt rules package. This action prevents real package installation make the tests much more predictable. ## Flaky test runner - ✅ [ECH import custom rules integration tests (200 runs)](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8627) - ✅ [Serverless import custom rules integration tests (200 runs)](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/8642)
1 parent 3102200 commit 76908c5

15 files changed

Lines changed: 113 additions & 84 deletions

File tree

‎src/platform/packages/shared/kbn-test/src/functional_test_runner/functional_test_runner.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ export class FunctionalTestRunner {
9393
log: this.log,
9494
config: this.config,
9595
providers,
96+
// Skipping root hooks when there are no tests to execute
97+
skipRootHooks: !realServices,
9698
esVersion: this.esVersion,
9799
reporter,
98100
reporterOptions,
@@ -162,6 +164,7 @@ export class FunctionalTestRunner {
162164
log: this.log,
163165
config: this.config,
164166
providers,
167+
skipRootHooks: true,
165168
esVersion: this.esVersion,
166169
});
167170

‎src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/config/schema.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,13 @@ export const schema = Joi.object()
151151
slow: Joi.number().default(30000),
152152
timeout: Joi.number().default(INSPECTING ? 360000 * 100 : 360000),
153153
ui: Joi.string().default('bdd'),
154+
// Currently supporting beforeAll and afterAll.
155+
rootHooks: Joi.object()
156+
.keys({
157+
beforeAll: Joi.function().optional(),
158+
afterAll: Joi.function().optional(),
159+
})
160+
.optional(),
154161
})
155162
.default(),
156163

‎src/platform/packages/shared/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ interface Options {
3131
config: Config;
3232
providers: ProviderCollection;
3333
esVersion: EsVersion;
34+
skipRootHooks?: boolean;
3435
reporter?: any;
3536
reporterOptions?: any;
3637
}
@@ -45,12 +46,21 @@ export async function setupMocha({
4546
config,
4647
providers,
4748
esVersion,
49+
skipRootHooks,
4850
reporter,
4951
reporterOptions,
5052
}: Options) {
53+
const rootHooks = config.get('mochaOpts.rootHooks');
54+
5155
// configure mocha
5256
const mocha = new Mocha({
5357
...config.get('mochaOpts'),
58+
rootHooks: {
59+
beforeAll:
60+
rootHooks?.beforeAll && !skipRootHooks ? () => rootHooks.beforeAll(providers) : undefined,
61+
afterAll:
62+
rootHooks?.afterAll && !skipRootHooks ? () => rootHooks.afterAll(providers) : undefined,
63+
},
5464
reporter:
5565
reporter || (await providers.loadExternalService('mocha reporter', MochaReporterProvider)),
5666
reporterOptions,
@@ -72,7 +82,7 @@ export async function setupMocha({
7282
paths: config.get('testFiles'),
7383
});
7484

75-
// valiate that there aren't any tests in multiple ciGroups
85+
// validate that there aren't any tests in multiple ciGroups
7686
validateCiGroupTags(log, mocha);
7787

7888
filterSuites({

‎x-pack/test/security_solution_api_integration/config/ess/config.base.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@
66
*/
77

88
import path from 'path';
9-
109
import { CA_CERT_PATH } from '@kbn/dev-utils';
1110
import { FtrConfigProviderContext, kbnTestConfig, kibanaTestUser } from '@kbn/test';
1211
import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
13-
import { services as baseServices } from './services';
1412
import { PRECONFIGURED_ACTION_CONNECTORS } from '../shared';
13+
import { installMockPrebuiltRulesPackage } from '../../test_suites/detections_response/utils';
14+
import { FtrProviderContext } from '../../ftr_provider_context';
15+
import { services as baseServices } from './services';
1516

1617
interface CreateTestConfigOptions {
1718
license: string;
@@ -127,6 +128,15 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s
127128
},
128129
mochaOpts: {
129130
grep: '/^(?!.*@skipInEss).*@ess.*/',
131+
rootHooks: {
132+
// Some of the Rule Management API endpoints install prebuilt rules package under the hood.
133+
// Prebuilt rules package installation has been known to be flakiness reason since
134+
// EPR might be unavailable or the network may have faults.
135+
// Real prebuilt rules package installation is prevented by
136+
// installing a lightweight mock package.
137+
beforeAll: ({ getService }: FtrProviderContext) =>
138+
installMockPrebuiltRulesPackage({ getService }),
139+
},
130140
},
131141
};
132142
};

‎x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,15 @@
77

88
import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
99
import { FtrConfigProviderContext } from '@kbn/test';
10+
import { FtrProviderContext } from '../../ftr_provider_context';
11+
import { installMockPrebuiltRulesPackage } from '../../test_suites/detections_response/utils';
12+
import { services } from './services';
1013
export interface CreateTestConfigOptions {
1114
testFiles: string[];
1215
junit: { reportName: string };
1316
kbnTestServerArgs?: string[];
1417
kbnTestServerEnv?: Record<string, string>;
1518
}
16-
import { services } from './services';
1719

1820
export function createTestConfig(options: CreateTestConfigOptions) {
1921
return async ({ readConfigFile }: FtrConfigProviderContext) => {
@@ -48,6 +50,15 @@ export function createTestConfig(options: CreateTestConfigOptions) {
4850
mochaOpts: {
4951
...svlSharedConfig.get('mochaOpts'),
5052
grep: '/^(?!.*@skipInServerless).*@serverless.*/',
53+
rootHooks: {
54+
// Some of the Rule Management API endpoints install prebuilt rules package under the hood.
55+
// Prebuilt rules package installation has been known to be flakiness reason since
56+
// EPR might be unavailable or the network may have faults.
57+
// Real prebuilt rules package installation is prevented by
58+
// installing a lightweight mock package.
59+
beforeAll: ({ getService }: FtrProviderContext) =>
60+
installMockPrebuiltRulesPackage({ getService }),
61+
},
5162
},
5263
};
5364
};

‎x-pack/test/security_solution_api_integration/config/serverless/config.base.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ import path from 'path';
88

99
import { FtrConfigProviderContext } from '@kbn/test';
1010
import { ScoutTestRunConfigCategory } from '@kbn/scout-info';
11-
import { services } from './services';
11+
import { FtrProviderContext } from '../../ftr_provider_context';
12+
import { installMockPrebuiltRulesPackage } from '../../test_suites/detections_response/utils';
1213
import { PRECONFIGURED_ACTION_CONNECTORS } from '../shared';
14+
import { services } from './services';
1315

1416
export interface CreateTestConfigOptions {
1517
testFiles: string[];
@@ -57,6 +59,15 @@ export function createTestConfig(options: CreateTestConfigOptions) {
5759
mochaOpts: {
5860
...svlSharedConfig.get('mochaOpts'),
5961
grep: '/^(?!.*(^|\\s)@skipInServerless(\\s|$)).*@serverless.*/',
62+
rootHooks: {
63+
// Some of the Rule Management API endpoints install prebuilt rules package under the hood.
64+
// Prebuilt rules package installation has been known to be flakiness reason since
65+
// EPR might be unavailable or the network may have faults.
66+
// Real prebuilt rules package installation is prevented by
67+
// installing a lightweight mock package.
68+
beforeAll: ({ getService }: FtrProviderContext) =>
69+
installMockPrebuiltRulesPackage({ getService }),
70+
},
6071
},
6172
};
6273
};

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_multiple_prebuilt_rules.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import {
1111
createRuleAssetSavedObject,
1212
deleteAllPrebuiltRuleAssets,
1313
getCustomQueryRuleParams,
14-
installMockPrebuiltRulesPackage,
1514
installPrebuiltRules,
1615
importRulesWithSuccess,
1716
} from '../../../../utils';
@@ -42,10 +41,6 @@ export default ({ getService }: FtrProviderContext): void => {
4241

4342
// These scenarios are "smoke tests" for all the user stories from the test plan's Product requirements section.
4443
describe('@ess @serverless @skipInServerlessMKI Import multiple prebuilt rules', () => {
45-
before(async () => {
46-
await installMockPrebuiltRulesPackage(es, supertest);
47-
});
48-
4944
beforeEach(async () => {
5045
await deleteAllRules(supertest, log);
5146
await deleteAllPrebuiltRuleAssets(es, log);
@@ -55,11 +50,6 @@ export default ({ getService }: FtrProviderContext): void => {
5550
]);
5651
});
5752

58-
after(async () => {
59-
await deleteAllPrebuiltRuleAssets(es, log);
60-
await deleteAllRules(supertest, log);
61-
});
62-
6353
const NON_CUSTOMIZED_PREBUILT_RULE_TO_IMPORT = {
6454
...PREBUILT_RULE_ASSET_A['security-rule'],
6555
immutable: true,

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_outdated_prebuilt_rules.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import {
1010
createHistoricalPrebuiltRuleAssetSavedObjects,
1111
createRuleAssetSavedObject,
1212
deleteAllPrebuiltRuleAssets,
13-
installMockPrebuiltRulesPackage,
1413
installPrebuiltRules,
1514
importRulesWithSuccess,
1615
} from '../../../../utils';
@@ -61,10 +60,6 @@ export default ({ getService }: FtrProviderContext): void => {
6160
];
6261

6362
describe('@ess @serverless @skipInServerlessMKI Import multiple outdated prebuilt rules', () => {
64-
before(async () => {
65-
await installMockPrebuiltRulesPackage(es, supertest);
66-
});
67-
6863
beforeEach(async () => {
6964
await deleteAllRules(supertest, log);
7065
await deleteAllPrebuiltRuleAssets(es, log);
@@ -76,11 +71,6 @@ export default ({ getService }: FtrProviderContext): void => {
7671
]);
7772
});
7873

79-
after(async () => {
80-
await deleteAllPrebuiltRuleAssets(es, log);
81-
await deleteAllRules(supertest, log);
82-
});
83-
8474
describe('without overwriting', () => {
8575
it('imports outdated non-customized and customized prebuilt rules', async () => {
8676
const NEW_PREBUILT_RULE_ASSET_A = createRuleAssetSavedObject({

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_single_prebuilt_rule.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import {
1212
deleteAllPrebuiltRuleAssets,
1313
installPrebuiltRules,
1414
getCustomQueryRuleParams,
15-
installMockPrebuiltRulesPackage,
1615
reviewPrebuiltRulesToUpgrade,
1716
performUpgradePrebuiltRules,
1817
importRulesWithSuccess,
@@ -36,21 +35,12 @@ export default ({ getService }: FtrProviderContext): void => {
3635
});
3736

3837
describe('@ess @serverless @skipInServerlessMKI Import single prebuilt rule', () => {
39-
before(async () => {
40-
await installMockPrebuiltRulesPackage(es, supertest);
41-
});
42-
4338
beforeEach(async () => {
4439
await deleteAllRules(supertest, log);
4540
await deleteAllPrebuiltRuleAssets(es, log);
4641
await createHistoricalPrebuiltRuleAssetSavedObjects(es, [PREBUILT_RULE_ASSET]);
4742
});
4843

49-
after(async () => {
50-
await deleteAllPrebuiltRuleAssets(es, log);
51-
await deleteAllRules(supertest, log);
52-
});
53-
5444
describe('importing a single non-customized prebuilt rule', () => {
5545
const NON_CUSTOMIZED_PREBUILT_RULE_TO_IMPORT = {
5646
...PREBUILT_RULE_ASSET['security-rule'],

‎x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/import_export/import_with_installing_package.ts‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,11 +75,6 @@ export default ({ getService }: FtrProviderContext): void => {
7575
await deleteAllPrebuiltRuleAssets(es, log);
7676
});
7777

78-
after(async () => {
79-
await deleteAllPrebuiltRuleAssets(es, log);
80-
await deleteAllRules(supertest, log);
81-
});
82-
8378
const IMPORT_PAYLOAD = [
8479
{
8580
...NON_CUSTOMIZED_PREBUILT_RULE,
@@ -168,7 +163,7 @@ export default ({ getService }: FtrProviderContext): void => {
168163
// Package installation is rate limited. A single package installation is allowed per 10 seconds.
169164
await retryService.tryWithRetries(
170165
'installSecurityDetectionEnginePackage',
171-
async () => await installMockPrebuiltRulesPackage(es, supertest),
166+
async () => await installMockPrebuiltRulesPackage({ getService }),
172167
{
173168
retryCount: 5,
174169
retryDelay: 5000,

0 commit comments

Comments
 (0)