Skip to content

Commit 5fa2efc

Browse files
authored
Merge branch 'main' into graph-roles
2 parents 4d82f67 + efa2d14 commit 5fa2efc

3,290 files changed

Lines changed: 90709 additions & 25276 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml

Lines changed: 0 additions & 44 deletions
This file was deleted.

.buildkite/pipeline-resource-definitions/locations.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ spec:
2525
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-deploy-project.yml
2626
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-entity-store-performance-from-pr.yml
2727
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-2.yml
28-
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
2928
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
3029
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
3130
- https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-docs-sync.yml
Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
import { load as yamlLoad } from 'js-yaml';
11+
import {
12+
getAgentImageConfig,
13+
expandAgentQueue,
14+
DEFAULT_AGENT_IMAGE_CONFIG,
15+
ELASTIC_IMAGES_QA_PROJECT,
16+
FIPS_140_3_IMAGE,
17+
FIPS_140_2_IMAGE,
18+
USE_QA_IMAGE_GH_LABEL,
19+
} from './agent_images';
20+
import { FIPS_GH_LABELS, FIPS_VERSION } from './pr_labels';
21+
22+
const mockSetAnnotation = jest.fn();
23+
24+
jest.mock('./buildkite', () => {
25+
const actual = jest.requireActual('./buildkite');
26+
return {
27+
...actual,
28+
BuildkiteClient: jest.fn().mockImplementation(() => ({
29+
setAnnotation: mockSetAnnotation,
30+
})),
31+
};
32+
});
33+
34+
const ORIGINAL_ENV = process.env;
35+
36+
describe('agent_images', () => {
37+
beforeEach(() => {
38+
jest.clearAllMocks();
39+
process.env = { ...ORIGINAL_ENV };
40+
delete process.env.GITHUB_PR_LABELS;
41+
delete process.env.TEST_ENABLE_FIPS_VERSION;
42+
delete process.env.USE_QA_IMAGE_FOR_PR;
43+
});
44+
45+
afterAll(() => {
46+
process.env = ORIGINAL_ENV;
47+
});
48+
49+
describe('getAgentImageConfig', () => {
50+
it('returns default config when no FIPS or QA env is set', () => {
51+
const config = getAgentImageConfig();
52+
53+
expect(config).toEqual(DEFAULT_AGENT_IMAGE_CONFIG);
54+
expect(mockSetAnnotation).not.toHaveBeenCalled();
55+
});
56+
57+
it('returns FIPS 140-2 image when env is set', () => {
58+
process.env.TEST_ENABLE_FIPS_VERSION = FIPS_VERSION.TWO;
59+
const config = getAgentImageConfig();
60+
61+
expect(config).toEqual(
62+
expect.objectContaining({
63+
image: expect.stringContaining(FIPS_140_2_IMAGE),
64+
imageProject: DEFAULT_AGENT_IMAGE_CONFIG.imageProject,
65+
})
66+
);
67+
expect(mockSetAnnotation).toHaveBeenCalledWith(
68+
'agent image config',
69+
'info',
70+
expect.stringContaining('FIPS Agents Enabled')
71+
);
72+
});
73+
74+
it('returns FIPS 140-3 image when env is set', () => {
75+
process.env.TEST_ENABLE_FIPS_VERSION = FIPS_VERSION.THREE;
76+
const config = getAgentImageConfig();
77+
78+
expect(config).toEqual(
79+
expect.objectContaining({
80+
image: expect.stringContaining(FIPS_140_3_IMAGE),
81+
})
82+
);
83+
expect(mockSetAnnotation).toHaveBeenCalled();
84+
});
85+
86+
it('returns FIPS image when FIPS label is present', () => {
87+
process.env.GITHUB_PR_LABELS = FIPS_GH_LABELS[FIPS_VERSION.TWO];
88+
const config = getAgentImageConfig();
89+
90+
expect(config).toEqual(
91+
expect.objectContaining({
92+
image: expect.stringContaining(FIPS_140_2_IMAGE),
93+
})
94+
);
95+
expect(mockSetAnnotation).toHaveBeenCalled();
96+
});
97+
98+
it('uses QA image project when env is set', () => {
99+
process.env.USE_QA_IMAGE_FOR_PR = 'true';
100+
const config = getAgentImageConfig();
101+
102+
expect(config).toEqual(
103+
expect.objectContaining({
104+
imageProject: ELASTIC_IMAGES_QA_PROJECT,
105+
})
106+
);
107+
});
108+
109+
it('uses QA image project when label is set', () => {
110+
process.env.GITHUB_PR_LABELS = USE_QA_IMAGE_GH_LABEL;
111+
const config = getAgentImageConfig();
112+
113+
expect(config).toEqual(
114+
expect.objectContaining({
115+
imageProject: ELASTIC_IMAGES_QA_PROJECT,
116+
})
117+
);
118+
});
119+
120+
it('returns valid YAML when returnYaml is true', () => {
121+
const yaml = getAgentImageConfig({ returnYaml: true });
122+
123+
expect(typeof yaml).toBe('string');
124+
const parsed = yamlLoad(yaml) as Record<string, unknown>;
125+
expect(parsed).toHaveProperty('agents');
126+
const agents = parsed.agents as Record<string, unknown>;
127+
expect(agents).toHaveProperty('provider', DEFAULT_AGENT_IMAGE_CONFIG.provider);
128+
});
129+
});
130+
131+
describe('expandAgentQueue', () => {
132+
it('returns spot config for default queue', () => {
133+
const config = expandAgentQueue();
134+
135+
expect(config).toEqual(
136+
expect.objectContaining({
137+
machineType: 'n2-standard-4',
138+
preemptible: true,
139+
provider: DEFAULT_AGENT_IMAGE_CONFIG.provider,
140+
})
141+
);
142+
});
143+
144+
it('returns virt config for virt queue', () => {
145+
const config = expandAgentQueue('n2-4-virt');
146+
147+
expect(config).toEqual(
148+
expect.objectContaining({
149+
machineType: 'n2-standard-4',
150+
enableNestedVirtualization: true,
151+
})
152+
);
153+
});
154+
155+
it('uses custom disk size when provided', () => {
156+
const config = expandAgentQueue('n2-4-spot', 200);
157+
158+
expect(config.diskSizeGb).toBe(200);
159+
});
160+
161+
it('returns base config for queue without spot or virt suffix', () => {
162+
const config = expandAgentQueue('c2-8');
163+
164+
expect(config).toEqual(
165+
expect.objectContaining({
166+
machineType: 'c2-standard-8',
167+
provider: DEFAULT_AGENT_IMAGE_CONFIG.provider,
168+
})
169+
);
170+
expect(config).not.toHaveProperty('preemptible');
171+
expect(config).not.toHaveProperty('enableNestedVirtualization');
172+
});
173+
});
174+
});

.buildkite/pipeline-utils/agent_images.ts

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,40 +12,35 @@ import type { BuildkiteAgentTargetingRule } from './buildkite';
1212
import { BuildkiteClient } from './buildkite';
1313
import { FIPS_VERSION, prHasFIPSLabel } from './pr_labels';
1414

15-
const ELASTIC_IMAGES_QA_PROJECT = 'elastic-images-qa';
16-
const ELASTIC_IMAGES_PROD_PROJECT = 'elastic-images-prod';
15+
export const ELASTIC_IMAGES_QA_PROJECT = 'elastic-images-qa';
16+
export const USE_QA_IMAGE_GH_LABEL = 'ci:use-qa-image';
17+
export const ELASTIC_IMAGES_PROD_PROJECT = 'elastic-images-prod';
18+
export const FIPS_140_3_IMAGE = 'family/kibana-fips-140-3-ubuntu-2404';
19+
export const FIPS_140_2_IMAGE = 'family/kibana-fips-140-2-ubuntu-2404';
1720

1821
// constrain AgentImageConfig to the type that doesn't have the `queue` property
19-
const DEFAULT_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
22+
export const DEFAULT_AGENT_IMAGE_CONFIG: BuildkiteAgentTargetingRule = {
2023
provider: 'gcp',
2124
image: 'family/kibana-ubuntu-2404',
2225
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
2326
diskSizeGb: 105,
2427
};
2528

26-
const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
27-
const USE_FIPS_IMAGE_FOR_PR = process.env.TEST_ENABLE_FIPS_VERSION?.match(
28-
new RegExp(`^${FIPS_VERSION.TWO}|${FIPS_VERSION.THREE}$`)
29-
);
30-
const USE_QA_IMAGE_FOR_PR = process.env.USE_QA_IMAGE_FOR_PR?.match(/(1|true)/i);
31-
3229
const getFIPSImage = () => {
3330
let image: string;
3431

3532
if (
3633
process.env.TEST_ENABLE_FIPS_VERSION === FIPS_VERSION.THREE ||
3734
prHasFIPSLabel(FIPS_VERSION.THREE)
3835
) {
39-
image = 'family/kibana-fips-140-3-ubuntu-2404';
36+
image = FIPS_140_3_IMAGE;
4037
} else {
41-
image = 'family/kibana-fips-140-2-ubuntu-2404';
38+
image = FIPS_140_2_IMAGE;
4239
}
4340

4441
return {
45-
provider: 'gcp',
42+
...DEFAULT_AGENT_IMAGE_CONFIG,
4643
image,
47-
imageProject: ELASTIC_IMAGES_PROD_PROJECT,
48-
diskSizeGb: 105,
4944
};
5045
};
5146

@@ -54,9 +49,15 @@ function getAgentImageConfig(): BuildkiteAgentTargetingRule;
5449
function getAgentImageConfig(options: { returnYaml: true }): string;
5550
function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAgentTargetingRule {
5651
const bk = new BuildkiteClient();
52+
const prLabels = process.env.GITHUB_PR_LABELS ?? '';
53+
const useFipsImage = process.env.TEST_ENABLE_FIPS_VERSION?.match(
54+
new RegExp(`^${FIPS_VERSION.TWO}|${FIPS_VERSION.THREE}$`)
55+
);
56+
const useQaImage =
57+
process.env.USE_QA_IMAGE_FOR_PR?.match(/(1|true)/i) || prLabels.includes(USE_QA_IMAGE_GH_LABEL);
5758
let config: BuildkiteAgentTargetingRule;
5859

59-
if (USE_FIPS_IMAGE_FOR_PR || prHasFIPSLabel()) {
60+
if (useFipsImage || prHasFIPSLabel()) {
6061
config = getFIPSImage();
6162

6263
bk.setAnnotation(
@@ -65,11 +66,11 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | BuildkiteAge
6566
'#### FIPS Agents Enabled<br />\nFIPS mode can produce new test failures. If you did not intend this remove ```TEST_ENABLE_FIPS_VERSION``` environment variable and/or the ```ci:enable-fips-<version>-agent``` Github label.'
6667
);
6768
} else {
68-
config = DEFAULT_AGENT_IMAGE_CONFIG;
69+
config = { ...DEFAULT_AGENT_IMAGE_CONFIG };
6970
}
7071

71-
if (USE_QA_IMAGE_FOR_PR || GITHUB_PR_LABELS.includes('ci:use-qa-image')) {
72-
config.imageProject = ELASTIC_IMAGES_QA_PROJECT;
72+
if (useQaImage) {
73+
config = { ...config, imageProject: ELASTIC_IMAGES_QA_PROJECT };
7374
}
7475

7576
if (returnYaml) {

.buildkite/pipelines/chrome_forward_testing.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ steps:
262262
depends_on:
263263
- build
264264
timeout_in_minutes: 60
265-
parallelism: 1
265+
parallelism: 2
266266
retry:
267267
automatic:
268268
- exit_status: '-1'

.buildkite/pipelines/evals/llm_evals.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
env:
22
KBN_EVALS: '1'
3+
KBN_EVALS_WEEKLY: '1'
34
steps:
45
- label: '👨‍🔧 Pre-Build'
56
command: .buildkite/scripts/lifecycle/pre_build.sh
@@ -220,7 +221,7 @@ steps:
220221
- exit_status: '-1'
221222
limit: 3
222223

223-
- label: 'Evals: Entity Anlaytics'
224+
- label: 'Evals: Entity Analytics'
224225
key: kbn-evals-weekly-entity-analytics
225226
command: bash .buildkite/scripts/steps/evals/run_suite.sh
226227
env:

.buildkite/pipelines/node_glibc_217.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ steps:
283283
depends_on:
284284
- build
285285
timeout_in_minutes: 60
286-
parallelism: 1
286+
parallelism: 2
287287
retry:
288288
automatic:
289289
- exit_status: '-1'

.buildkite/pipelines/node_pointer_compression.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ steps:
285285
depends_on:
286286
- build
287287
timeout_in_minutes: 60
288-
parallelism: 1
288+
parallelism: 2
289289
retry:
290290
automatic:
291291
- exit_status: '-1'
Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
steps:
2-
# - command: .buildkite/scripts/steps/checks/api_contracts.sh
3-
# label: 'Check API Contracts'
4-
# agents:
5-
# machineType: c4d-standard-2
6-
# diskType: hyperdisk-balanced
7-
# preemptible: true
8-
# spotZones: us-central1-c,us-central1-a
9-
# key: check_api_contracts
10-
# depends_on: check_oas_snapshot
11-
# timeout_in_minutes: 30
12-
# soft_fail: true
13-
# retry:
14-
# automatic:
15-
# - exit_status: '-1'
16-
# limit: 3
2+
- command: .buildkite/scripts/steps/checks/api_contracts.sh
3+
label: 'Check API Contracts'
4+
agents:
5+
machineType: c4d-standard-2
6+
diskType: hyperdisk-balanced
7+
preemptible: true
8+
spotZones: us-central1-c,us-central1-a
9+
key: check_api_contracts
10+
depends_on: check_oas_snapshot
11+
timeout_in_minutes: 30
12+
soft_fail: true
13+
retry:
14+
automatic:
15+
- exit_status: '-1'
16+
limit: 3

0 commit comments

Comments
 (0)