Skip to content

Commit f2cba71

Browse files
pjhamptonkibanamachine
pjhampton
and
kibanamachine
committed
Endpoint Telemetry: Agents Metrics + Policy Config / Response (#102171)
* [PH] Initial setup for endpoint task telemetry. * Refactor / Add daily task for collecting fleet detail / policy resp / EP metrics * [PH CD] Code walkthrough. Start fetching fleet policy configs. * [PH] pass in fleet agent service rather than homebrew kuerys. * [PH] prepare to move away from legacy es client. Get fleet ep agents. * Fetch agent policy configs. * Stub ep policy responses. * Fix CI + Types. Fix dep injection. Reimagine SO client creation. * Create SO client properly * Fetch EP Policy responses. * Fetch EP Policy responses. * Remove unused import * Fetch failed policy responses from EP data stream. * Remove unused imports. * Combine failed policy responses with policy configs. * Attach fleet agent + ep agent ids * Add dedicated channel sender. Temp disable with feature flag. * Remove ublock from the failed policy response. * Fetch endpoint metrics. * Fix bad merge commit. * Get EP telemetry. * Record last execution time of endpoint task * Remove send on demand feature flag. * Simplify cache conditional. * Refactor into Promise.allSettled * Fix type error. * Bail if there is no endpoint metrics * Bump interval to 24h. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
1 parent e6c0fb6 commit f2cba71

10 files changed

Lines changed: 649 additions & 83 deletions

File tree

x-pack/plugins/security_solution/server/lib/telemetry/task.test.ts renamed to x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.test.ts

Lines changed: 5 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,10 @@
55
* 2.0.
66
*/
77

8-
import moment from 'moment';
98
import { loggingSystemMock } from 'src/core/server/mocks';
10-
119
import { taskManagerMock } from '../../../../task_manager/server/mocks';
1210
import { TaskStatus } from '../../../../task_manager/server';
13-
14-
import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './task';
11+
import { TelemetryDiagTask, TelemetryDiagTaskConstants } from './diagnostic_task';
1512
import { createMockTelemetryEventsSender, MockTelemetryDiagnosticTask } from './mocks';
1613

1714
describe('test', () => {
@@ -22,7 +19,7 @@ describe('test', () => {
2219
});
2320

2421
describe('basic diagnostic alert telemetry sanity checks', () => {
25-
test('task can register', () => {
22+
test('diagnostic task can register', () => {
2623
const telemetryDiagTask = new TelemetryDiagTask(
2724
logger,
2825
taskManagerMock.createSetup(),
@@ -40,7 +37,7 @@ describe('test', () => {
4037
expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled();
4138
});
4239

43-
test('task should be scheduled', async () => {
40+
test('diagnostic task should be scheduled', async () => {
4441
const mockTaskManagerSetup = taskManagerMock.createSetup();
4542
const telemetryDiagTask = new TelemetryDiagTask(
4643
logger,
@@ -53,7 +50,7 @@ describe('test', () => {
5350
expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled();
5451
});
5552

56-
test('task should run', async () => {
53+
test('diagnostic task should run', async () => {
5754
const mockContext = createMockTelemetryEventsSender(true);
5855
const mockTaskManager = taskManagerMock.createSetup();
5956
const telemetryDiagTask = new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockContext);
@@ -79,7 +76,7 @@ describe('test', () => {
7976
expect(telemetryDiagTask.runTask).toHaveBeenCalled();
8077
});
8178

82-
test('task should not query elastic if telemetry is not opted in', async () => {
79+
test('diagnostic task should not query elastic if telemetry is not opted in', async () => {
8380
const mockSender = createMockTelemetryEventsSender(false);
8481
const mockTaskManager = taskManagerMock.createSetup();
8582
new MockTelemetryDiagnosticTask(logger, mockTaskManager, mockSender);
@@ -104,48 +101,4 @@ describe('test', () => {
104101
await taskRunner.run();
105102
expect(mockSender.fetchDiagnosticAlerts).not.toHaveBeenCalled();
106103
});
107-
108-
test('test -5 mins is returned when there is no previous task run', async () => {
109-
const telemetryDiagTask = new TelemetryDiagTask(
110-
logger,
111-
taskManagerMock.createSetup(),
112-
createMockTelemetryEventsSender(true)
113-
);
114-
115-
const executeTo = moment().utc().toISOString();
116-
const executeFrom = undefined;
117-
const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom);
118-
119-
expect(newExecuteFrom).toEqual(moment(executeTo).subtract(5, 'minutes').toISOString());
120-
});
121-
122-
test('test -6 mins is returned when there was a previous task run', async () => {
123-
const telemetryDiagTask = new TelemetryDiagTask(
124-
logger,
125-
taskManagerMock.createSetup(),
126-
createMockTelemetryEventsSender(true)
127-
);
128-
129-
const executeTo = moment().utc().toISOString();
130-
const executeFrom = moment(executeTo).subtract(6, 'minutes').toISOString();
131-
const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom);
132-
133-
expect(newExecuteFrom).toEqual(executeFrom);
134-
});
135-
136-
// it's possible if Kibana is down for a prolonged period the stored lastRun would have drifted
137-
// if that is the case we will just roll it back to a 10 min search window
138-
test('test 10 mins is returned when previous task run took longer than 10 minutes', async () => {
139-
const telemetryDiagTask = new TelemetryDiagTask(
140-
logger,
141-
taskManagerMock.createSetup(),
142-
createMockTelemetryEventsSender(true)
143-
);
144-
145-
const executeTo = moment().utc().toISOString();
146-
const executeFrom = moment(executeTo).subtract(142, 'minutes').toISOString();
147-
const newExecuteFrom = telemetryDiagTask.getLastExecutionTimestamp(executeTo, executeFrom);
148-
149-
expect(newExecuteFrom).toEqual(moment(executeTo).subtract(10, 'minutes').toISOString());
150-
});
151104
});

x-pack/plugins/security_solution/server/lib/telemetry/task.ts renamed to x-pack/plugins/security_solution/server/lib/telemetry/diagnostic_task.ts

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
TaskManagerSetupContract,
1313
TaskManagerStartContract,
1414
} from '../../../../task_manager/server';
15+
import { getLastTaskExecutionTimestamp } from './helpers';
1516
import { TelemetryEventsSender, TelemetryEvent } from './sender';
1617

1718
export const TelemetryDiagTaskConstants = {
@@ -43,7 +44,7 @@ export class TelemetryDiagTask {
4344
return {
4445
run: async () => {
4546
const executeTo = moment().utc().toISOString();
46-
const executeFrom = this.getLastExecutionTimestamp(
47+
const executeFrom = getLastTaskExecutionTimestamp(
4748
executeTo,
4849
taskInstance.state?.lastExecutionTimestamp
4950
);
@@ -64,20 +65,6 @@ export class TelemetryDiagTask {
6465
});
6566
}
6667

67-
public getLastExecutionTimestamp(executeTo: string, lastExecutionTimestamp?: string) {
68-
if (lastExecutionTimestamp === undefined) {
69-
this.logger.debug(`No last execution timestamp defined`);
70-
return moment(executeTo).subtract(5, 'minutes').toISOString();
71-
}
72-
73-
if (moment(executeTo).diff(lastExecutionTimestamp, 'minutes') >= 10) {
74-
this.logger.debug(`last execution timestamp was greater than 10 minutes`);
75-
return moment(executeTo).subtract(10, 'minutes').toISOString();
76-
}
77-
78-
return lastExecutionTimestamp;
79-
}
80-
8168
public start = async (taskManager: TaskManagerStartContract) => {
8269
try {
8370
await taskManager.ensureScheduled({

‎x-pack/plugins/security_solution/server/lib/telemetry/endpoint_task.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { loggingSystemMock } from 'src/core/server/mocks';
9+
import { taskManagerMock } from '../../../../task_manager/server/mocks';
10+
import { TelemetryEndpointTask } from './endpoint_task';
11+
import { createMockTelemetryEventsSender } from './mocks';
12+
13+
describe('test', () => {
14+
let logger: ReturnType<typeof loggingSystemMock.createLogger>;
15+
16+
beforeEach(() => {
17+
logger = loggingSystemMock.createLogger();
18+
});
19+
20+
describe('endpoint alert telemetry checks', () => {
21+
test('the task can register', () => {
22+
const telemetryEndpointTask = new TelemetryEndpointTask(
23+
logger,
24+
taskManagerMock.createSetup(),
25+
createMockTelemetryEventsSender(true)
26+
);
27+
28+
expect(telemetryEndpointTask).toBeInstanceOf(TelemetryEndpointTask);
29+
});
30+
});
31+
32+
test('the endpoint task should be registered', () => {
33+
const mockTaskManager = taskManagerMock.createSetup();
34+
new TelemetryEndpointTask(logger, mockTaskManager, createMockTelemetryEventsSender(true));
35+
36+
expect(mockTaskManager.registerTaskDefinitions).toHaveBeenCalled();
37+
});
38+
39+
test('the endpoint task should be scheduled', async () => {
40+
const mockTaskManagerSetup = taskManagerMock.createSetup();
41+
const telemetryEndpointTask = new TelemetryEndpointTask(
42+
logger,
43+
mockTaskManagerSetup,
44+
createMockTelemetryEventsSender(true)
45+
);
46+
47+
const mockTaskManagerStart = taskManagerMock.createStart();
48+
await telemetryEndpointTask.start(mockTaskManagerStart);
49+
expect(mockTaskManagerStart.ensureScheduled).toHaveBeenCalled();
50+
});
51+
});

0 commit comments

Comments
 (0)