Skip to content

Commit 28dbd60

Browse files
merge in main
2 parents 96e840b + 5fc1745 commit 28dbd60

1,306 files changed

Lines changed: 54768 additions & 7775 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import { load as yamlLoad } from 'js-yaml';
11+
import {
12+
getAgentImageConfig,
13+
expandAgentQueue,
14+
DEFAULT_AGENT_IMAGE_CONFIG,
15+
ELASTIC_IMAGES_QA_PROJECT,
16+
FIPS_140_3_IMAGE,
17+
FIPS_140_2_IMAGE,
18+
USE_QA_IMAGE_GH_LABEL,
19+
} from './agent_images';
20+
import { FIPS_GH_LABELS, FIPS_VERSION } from './pr_labels';
21+
22+
const mockSetAnnotation = jest.fn();
23+
24+
jest.mock('./buildkite', () => {
25+
const actual = jest.requireActual('./buildkite');
26+
return {
27+
...actual,
28+
BuildkiteClient: jest.fn().mockImplementation(() => ({
29+
setAnnotation: mockSetAnnotation,
30+
})),
31+
};
32+
});
33+
34+
const ORIGINAL_ENV = process.env;
35+
36+
describe('agent_images', () => {
37+
beforeEach(() => {
38+
jest.clearAllMocks();
39+
process.env = { ...ORIGINAL_ENV };
40+
delete process.env.GITHUB_PR_LABELS;
41+
delete process.env.TEST_ENABLE_FIPS_VERSION;
42+
delete process.env.USE_QA_IMAGE_FOR_PR;
43+
});
44+
45+
afterAll(() => {
46+
process.env = ORIGINAL_ENV;
47+
});
48+
49+
describe('getAgentImageConfig', () => {
50+
it('returns default config when no FIPS or QA env is set', () => {
51+
const config = getAgentImageConfig();
52+
53+
expect(config).toEqual(DEFAULT_AGENT_IMAGE_CONFIG);
54+
expect(mockSetAnnotation).not.toHaveBeenCalled();
55+
});
56+
57+
it('returns FIPS 140-2 image when env is set', () => {
58+
process.env.TEST_ENABLE_FIPS_VERSION = FIPS_VERSION.TWO;
59+
const config = getAgentImageConfig();
60+
61+
expect(config).toEqual(
62+
expect.objectContaining({
63+
image: expect.stringContaining(FIPS_140_2_IMAGE),
64+
imageProject: DEFAULT_AGENT_IMAGE_CONFIG.imageProject,
65+
})
66+
);
67+
expect(mockSetAnnotation).toHaveBeenCalledWith(
68+
'agent image config',
69+
'info',
70+
expect.stringContaining('FIPS Agents Enabled')
71+
);
72+
});
73+
74+
it('returns FIPS 140-3 image when env is set', () => {
75+
process.env.TEST_ENABLE_FIPS_VERSION = FIPS_VERSION.THREE;
76+
const config = getAgentImageConfig();
77+
78+
expect(config).toEqual(
79+
expect.objectContaining({
80+
image: expect.stringContaining(FIPS_140_3_IMAGE),
81+
})
82+
);
83+
expect(mockSetAnnotation).toHaveBeenCalled();
84+
});
85+
86+
it('returns FIPS image when FIPS label is present', () => {
87+
process.env.GITHUB_PR_LABELS = FIPS_GH_LABELS[FIPS_VERSION.TWO];
88+
const config = getAgentImageConfig();
89+
90+
expect(config).toEqual(
91+
expect.objectContaining({
92+
image: expect.stringContaining(FIPS_140_2_IMAGE),
93+
})
94+
);
95+
expect(mockSetAnnotation).toHaveBeenCalled();
96+
});
97+
98+
it('uses QA image project when env is set', () => {
99+
process.env.USE_QA_IMAGE_FOR_PR = 'true';
100+
const config = getAgentImageConfig();
101+
102+
expect(config).toEqual(
103+
expect.objectContaining({
104+
imageProject: ELASTIC_IMAGES_QA_PROJECT,
105+
})
106+
);
107+
});
108+
109+
it('uses QA image project when label is set', () => {
110+
process.env.GITHUB_PR_LABELS = USE_QA_IMAGE_GH_LABEL;
111+
const config = getAgentImageConfig();
112+
113+
expect(config).toEqual(
114+
expect.objectContaining({
115+
imageProject: ELASTIC_IMAGES_QA_PROJECT,
116+
})
117+
);
118+
});
119+
120+
it('returns valid YAML when returnYaml is true', () => {
121+
const yaml = getAgentImageConfig({ returnYaml: true });
122+
123+
expect(typeof yaml).toBe('string');
124+
const parsed = yamlLoad(yaml) as Record<string, unknown>;
125+
expect(parsed).toHaveProperty('agents');
126+
const agents = parsed.agents as Record<string, unknown>;
127+
expect(agents).toHaveProperty('provider', DEFAULT_AGENT_IMAGE_CONFIG.provider);
128+
});
129+
});
130+
131+
describe('expandAgentQueue', () => {
132+
it('returns spot config for default queue', () => {
133+
const config = expandAgentQueue();
134+
135+
expect(config).toEqual(
136+
expect.objectContaining({
137+
machineType: 'n2-standard-4',
138+
preemptible: true,
139+
provider: DEFAULT_AGENT_IMAGE_CONFIG.provider,
140+
})
141+
);
142+
});
143+
144+
it('returns virt config for virt queue', () => {
145+
const config = expandAgentQueue('n2-4-virt');
146+
147+
expect(config).toEqual(
148+
expect.objectContaining({
149+
machineType: 'n2-standard-4',
150+
enableNestedVirtualization: true,
151+
})
152+
);
153+
});
154+
155+
it('uses custom disk size when provided', () => {
156+
const config = expandAgentQueue('n2-4-spot', 200);
157+
158+
expect(config.diskSizeGb).toBe(200);
159+
});
160+
161+
it('returns base config for queue without spot or virt suffix', () => {
162+
const config = expandAgentQueue('c2-8');
163+
164+
expect(config).toEqual(
165+
expect.objectContaining({
166+
machineType: 'c2-standard-8',
167+
provider: DEFAULT_AGENT_IMAGE_CONFIG.provider,
168+
})
169+
);
170+
expect(config).not.toHaveProperty('preemptible');
171+
expect(config).not.toHaveProperty('enableNestedVirtualization');
172+
});
173+
});
174+
});

.buildkite/pipeline-utils/agent_images.ts

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,40 +12,35 @@ import type { BuildkiteAgentTargetingRule } from './buildkite';
1212
import { BuildkiteClient } from './buildkite';
1313
import { FIPS_VERSION, prHasFIPSLabel } from './pr_labels';
1414

15-
const ELASTIC_IMAGES_QA_PROJECT = 'elastic-images-qa';
16-
const ELASTIC_IMAGES_PROD_PROJECT = 'elastic-images-prod';
15+
export const ELASTIC_IMAGES_QA_PROJECT = 'elastic-images-qa';
16+
export const USE_QA_IMAGE_GH_LABEL = 'ci:use-qa-image';
17+
export const ELASTIC_IMAGES_PROD_PROJECT = 'elastic-images-prod';
18+
export const FIPS_140_3_IMAGE = 'family/kibana-fips-140-3-ubuntu-2404';
19+
export const FIPS_140_2_IMAGE = 'family/kibana-fips-140-2-ubuntu-2404';
1720

1821
// constrain AgentImageConfig to the type that doesn't have the `queue` property
19-
const DEFAULT_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
22+
export const DEFAULT_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
2023
provider: 'gcp',
2124
image: 'family/kibana-ubuntu-2404',
2225
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
2326
diskSizeGb: 105,
2427
};
2528

26-
const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
27-
const USE_FIPS_IMAGE_FOR_PR = process.env.TEST_ENABLE_FIPS_VERSION?.match(
28-
new RegExp(`^${FIPS_VERSION.TWO}|${FIPS_VERSION.THREE}$`)
29-
);
30-
const USE_QA_IMAGE_FOR_PR = process.env.USE_QA_IMAGE_FOR_PR?.match(/(1|true)/i);
31-
3229
const getFIPSImage = () => {
3330
let image: string;
3431

3532
if (
3633
process.env.TEST_ENABLE_FIPS_VERSION === FIPS_VERSION.THREE ||
3734
prHasFIPSLabel(FIPS_VERSION.THREE)
3835
) {
39-
image = 'family/kibana-fips-140-3-ubuntu-2404';
36+
image = FIPS_140_3_IMAGE;
4037
} else {
41-
image = 'family/kibana-fips-140-2-ubuntu-2404';
38+
image = FIPS_140_2_IMAGE;
4239
}
4340

4441
return {
45-
provider: 'gcp',
42+
...DEFAULT_AGENT_IMAGE_CONFIG,
4643
image,
47-
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
48-
diskSizeGb: 105,
4944
};
5045
};
5146

@@ -54,9 +49,15 @@ function getAgentImageConfig(): BuildkiteAgentTargetingRule;
5449
function getAgentImageConfig(options: { returnYaml: true }): string;
5550
function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAgentTargetingRule {
5651
const bk = new BuildkiteClient();
52+
const prLabels = process.env.GITHUB_PR_LABELS ?? '';
53+
const useFipsImage = process.env.TEST_ENABLE_FIPS_VERSION?.match(
54+
new RegExp(`^${FIPS_VERSION.TWO}|${FIPS_VERSION.THREE}$`)
55+
);
56+
const useQaImage =
57+
process.env.USE_QA_IMAGE_FOR_PR?.match(/(1|true)/i) || prLabels.includes(USE_QA_IMAGE_GH_LABEL);
5758
let config: BuildkiteAgentTargetingRule;
5859

59-
if (USE_FIPS_IMAGE_FOR_PR || prHasFIPSLabel()) {
60+
if (useFipsImage || prHasFIPSLabel()) {
6061
config = getFIPSImage();
6162

6263
bk.setAnnotation(
@@ -65,11 +66,11 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAge
6566
'#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```TEST_ENABLE_FIPS_VERSION``` environment variable and/or the ```ci:enable-fips-<version>-agent``` Github label.'
6667
);
6768
} else {
68-
config = DEFAULT_AGENT_IMAGE_CONFIG;
69+
config = { ...DEFAULT_AGENT_IMAGE_CONFIG };
6970
}
7071

71-
if (USE_QA_IMAGE_FOR_PR || GITHUB_PR_LABELS.includes('ci:use-qa-image')) {
72-
config.imageProject = ELASTIC_IMAGES_QA_PROJECT;
72+
if (useQaImage) {
73+
config = { ...config, imageProject: ELASTIC_IMAGES_QA_PROJECT };
7374
}
7475

7576
if (returnYaml) {

.buildkite/pipelines/chrome_forward_testing.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ steps:
262262
depends_on:
263263
- build
264264
timeout_in_minutes: 60
265-
parallelism: 1
265+
parallelism: 2
266266
retry:
267267
automatic:
268268
- exit_status: '-1'

.buildkite/pipelines/node_glibc_217.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ steps:
283283
depends_on:
284284
- build
285285
timeout_in_minutes: 60
286-
parallelism: 1
286+
parallelism: 2
287287
retry:
288288
automatic:
289289
- exit_status: '-1'

.buildkite/pipelines/node_pointer_compression.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ steps:
285285
depends_on:
286286
- build
287287
timeout_in_minutes: 60
288-
parallelism: 1
288+
parallelism: 2
289289
retry:
290290
automatic:
291291
- exit_status: '-1'

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_detection_engine.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ steps:
1414
machineType: n2-standard-4
1515
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
1616
timeout_in_minutes: 300
17-
parallelism: 3
17+
parallelism: 2
1818

1919
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:detection_alerts
2020
label: 'Cypress MKI - Detection Alerts'
@@ -42,7 +42,7 @@ steps:
4242
machineType: n2-standard-4
4343
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
4444
timeout_in_minutes: 300
45-
parallelism: 4
45+
parallelism: 3
4646

4747
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:overview
4848
label: 'Cypress MKI - Overview'
@@ -308,6 +308,7 @@ steps:
308308
provider: gcp
309309
machineType: n2-standard-4
310310
timeout_in_minutes: 120
311+
parallelism: 2
311312
retry:
312313
automatic:
313314
- exit_status: '1'
@@ -350,6 +351,7 @@ steps:
350351
provider: gcp
351352
machineType: n2-standard-4
352353
timeout_in_minutes: 120
354+
parallelism: 2
353355
retry:
354356
automatic:
355357
- exit_status: '1'

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_entity_analytics.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ steps:
1414
machineType: n2-standard-4
1515
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
1616
timeout_in_minutes: 300
17-
parallelism: 2
17+
parallelism: 1
1818

1919
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:entity_analytics:dashboards
2020
label: 'Cypress MKI - Dashboards'

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_explore.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ steps:
3131
machineType: n2-standard-4
3232
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
3333
timeout_in_minutes: 300
34-
parallelism: 1
34+
parallelism: 2
3535

3636
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore:cases
3737
label: "Cypress MKI - Cases"

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_investigations.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ steps:
1414
machineType: n2-standard-4
1515
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
1616
timeout_in_minutes: 300
17-
parallelism: 4
17+
parallelism: 5
1818

1919
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations:dashboards
2020
label: 'Cypress MKI - Dashboards'
@@ -98,7 +98,7 @@ steps:
9898
machineType: n2-standard-4
9999
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
100100
timeout_in_minutes: 300
101-
parallelism: 2
101+
parallelism: 3
102102

103103
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations:filters
104104
label: 'Cypress MKI - Explore - Filters'

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_rule_management.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ steps:
5656
machineType: n2-standard-4
5757
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
5858
timeout_in_minutes: 300
59-
parallelism: 3
59+
parallelism: 1
6060

6161
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:related_integrations
6262
label: 'Cypress MKI - Related Integrations'
@@ -84,7 +84,7 @@ steps:
8484
machineType: n2-standard-4
8585
# TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
8686
timeout_in_minutes: 300
87-
parallelism: 2
87+
parallelism: 1
8888

8989
- command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:rule_details
9090
label: 'Cypress MKI - Rule Details'

0 commit comments

Comments
 (0)