Skip to content

Commit 7cd2b6c

Browse files
afharokibanamachine
afharo
and
kibanamachine
authored
[7.x] [Telemetry] Move Monitoring collection strategy to a collector (#82638) (#83686)
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
1 parent fa52b06 commit 7cd2b6c

17 files changed

Lines changed: 356 additions & 270 deletions

File tree

‎src/plugins/telemetry_collection_manager/server/plugin.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,9 +289,9 @@ export class TelemetryCollectionManagerPlugin
289289
return stats.map((stat) => {
290290
const license = licenses[stat.cluster_uuid];
291291
return {
292+
collectionSource: collection.title,
292293
...(license ? { license } : {}),
293294
...stat,
294-
collectionSource: collection.title,
295295
};
296296
});
297297
}

‎x-pack/plugins/monitoring/kibana.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
],
1414
"optionalPlugins": [
1515
"infra",
16-
"telemetryCollectionManager",
1716
"usageCollection",
1817
"home",
1918
"cloud",

‎x-pack/plugins/monitoring/server/plugin.ts‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ import {
2020
CoreStart,
2121
CustomHttpResponseOptions,
2222
ResponseError,
23-
IClusterClient,
24-
SavedObjectsServiceStart,
2523
} from 'kibana/server';
2624
import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
2725
import {
@@ -41,7 +39,7 @@ import { initInfraSource } from './lib/logs/init_infra_source';
4139
import { mbSafeQuery } from './lib/mb_safe_query';
4240
import { instantiateClient } from './es_client/instantiate_client';
4341
import { registerCollectors } from './kibana_monitoring/collectors';
44-
import { registerMonitoringCollection } from './telemetry_collection';
42+
import { registerMonitoringTelemetryCollection } from './telemetry_collection';
4543
import { LicenseService } from './license_service';
4644
import { AlertsFactory } from './alerts';
4745
import {
@@ -77,8 +75,6 @@ export class Plugin {
7775
private monitoringCore = {} as MonitoringCore;
7876
private legacyShimDependencies = {} as LegacyShimDependencies;
7977
private bulkUploader: IBulkUploader = {} as IBulkUploader;
80-
private telemetryElasticsearchClient: IClusterClient | undefined;
81-
private telemetrySavedObjectsService: SavedObjectsServiceStart | undefined;
8278

8379
constructor(initializerContext: PluginInitializerContext) {
8480
this.initializerContext = initializerContext;
@@ -148,19 +144,6 @@ export class Plugin {
148144
plugins.alerts?.registerType(alert.getAlertType());
149145
}
150146

151-
// Initialize telemetry
152-
if (plugins.telemetryCollectionManager) {
153-
registerMonitoringCollection({
154-
telemetryCollectionManager: plugins.telemetryCollectionManager,
155-
esCluster: this.cluster,
156-
esClientGetter: () => this.telemetryElasticsearchClient,
157-
soServiceGetter: () => this.telemetrySavedObjectsService,
158-
customContext: {
159-
maxBucketSize: config.ui.max_bucket_size,
160-
},
161-
});
162-
}
163-
164147
// Register collector objects for stats to show up in the APIs
165148
if (plugins.usageCollection) {
166149
core.savedObjects.registerType({
@@ -177,6 +160,11 @@ export class Plugin {
177160
});
178161

179162
registerCollectors(plugins.usageCollection, config, cluster);
163+
registerMonitoringTelemetryCollection(
164+
plugins.usageCollection,
165+
cluster,
166+
config.ui.max_bucket_size
167+
);
180168
}
181169

182170
// Always create the bulk uploader
@@ -256,16 +244,7 @@ export class Plugin {
256244
};
257245
}
258246

259-
start({ elasticsearch, savedObjects }: CoreStart) {
260-
// TODO: For the telemetry plugin to work, we need to provide the new ES client.
261-
// The new client should be inititalized with a similar config to `this.cluster` but, since we're not using
262-
// the new client in Monitoring Telemetry collection yet, setting the local client allows progress for now.
263-
// The usage collector `fetch` method has been refactored to accept a `collectorFetchContext` object,
264-
// exposing both es clients and the saved objects client.
265-
// We will update the client in a follow up PR.
266-
this.telemetryElasticsearchClient = elasticsearch.client;
267-
this.telemetrySavedObjectsService = savedObjects;
268-
}
247+
start() {}
269248

270249
stop() {
271250
if (this.cluster) {

‎x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts‎

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,10 @@ import { getStackStats, getAllStats, handleAllStats } from './get_all_stats';
99
import { ESClusterStats } from './get_es_stats';
1010
import { KibanaStats } from './get_kibana_stats';
1111
import { ClustersHighLevelStats } from './get_high_level_stats';
12-
import { coreMock } from 'src/core/server/mocks';
1312

1413
describe('get_all_stats', () => {
1514
const timestamp = Date.now();
1615
const callCluster = sinon.stub();
17-
const esClient = sinon.stub();
18-
const soClient = sinon.stub();
1916

2017
const esClusters = [
2118
{ cluster_uuid: 'a' },
@@ -172,24 +169,7 @@ describe('get_all_stats', () => {
172169
.onCall(4)
173170
.returns(Promise.resolve({})); // Beats state
174171

175-
expect(
176-
await getAllStats(
177-
[{ clusterUuid: 'a' }],
178-
{
179-
callCluster: callCluster as any,
180-
esClient: esClient as any,
181-
soClient: soClient as any,
182-
usageCollection: {} as any,
183-
kibanaRequest: undefined,
184-
timestamp,
185-
},
186-
{
187-
logger: coreMock.createPluginInitializerContext().logger.get('test'),
188-
version: 'version',
189-
maxBucketSize: 1,
190-
}
191-
)
192-
).toStrictEqual(allClusters);
172+
expect(await getAllStats(['a'], callCluster, timestamp, 1)).toStrictEqual(allClusters);
193173
});
194174

195175
it('returns empty clusters', async () => {
@@ -199,24 +179,7 @@ describe('get_all_stats', () => {
199179

200180
callCluster.withArgs('search').returns(Promise.resolve(clusterUuidsResponse));
201181

202-
expect(
203-
await getAllStats(
204-
[],
205-
{
206-
callCluster: callCluster as any,
207-
esClient: esClient as any,
208-
soClient: soClient as any,
209-
usageCollection: {} as any,
210-
kibanaRequest: undefined,
211-
timestamp,
212-
},
213-
{
214-
logger: coreMock.createPluginInitializerContext().logger.get('test'),
215-
version: 'version',
216-
maxBucketSize: 1,
217-
}
218-
)
219-
).toStrictEqual([]);
182+
expect(await getAllStats([], callCluster, timestamp, 1)).toStrictEqual([]);
220183
});
221184
});
222185

‎x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
import { set } from '@elastic/safer-lodash-set';
88
import { get, merge } from 'lodash';
99

10-
import { StatsGetter } from 'src/plugins/telemetry_collection_manager/server';
1110
import moment from 'moment';
11+
import { LegacyAPICaller } from 'kibana/server';
1212
import {
1313
LOGSTASH_SYSTEM_ID,
1414
KIBANA_SYSTEM_ID,
@@ -20,24 +20,20 @@ import { getKibanaStats, KibanaStats } from './get_kibana_stats';
2020
import { getBeatsStats, BeatsStatsByClusterUuid } from './get_beats_stats';
2121
import { getHighLevelStats, ClustersHighLevelStats } from './get_high_level_stats';
2222

23-
export interface CustomContext {
24-
maxBucketSize: number;
25-
}
2623
/**
2724
* Get statistics for all products joined by Elasticsearch cluster.
2825
* Returns the array of clusters joined with the Kibana and Logstash instances.
2926
*
3027
*/
31-
export const getAllStats: StatsGetter<CustomContext> = async (
32-
clustersDetails,
33-
{ callCluster, timestamp },
34-
{ maxBucketSize }
35-
) => {
28+
export async function getAllStats(
29+
clusterUuids: string[],
30+
callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates
31+
timestamp: number,
32+
maxBucketSize: number
33+
) {
3634
const start = moment(timestamp).subtract(USAGE_FETCH_INTERVAL, 'ms').toISOString();
3735
const end = moment(timestamp).toISOString();
3836

39-
const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid);
40-
4137
const [esClusters, kibana, logstash, beats] = await Promise.all([
4238
getElasticsearchStats(callCluster, clusterUuids, maxBucketSize), // cluster_stats, stack_stats.xpack, cluster_name/uuid, license, version
4339
getKibanaStats(callCluster, clusterUuids, start, end, maxBucketSize), // stack_stats.kibana
@@ -46,7 +42,7 @@ export const getAllStats: StatsGetter<CustomContext> = async (
4642
]);
4743

4844
return handleAllStats(esClusters, { kibana, logstash, beats });
49-
};
45+
}
5046

5147
/**
5248
* Combine the statistics from the stack to create "cluster" stats that associate all products together based on the cluster

‎x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,55 +5,35 @@
55
*/
66

77
import sinon from 'sinon';
8-
import { elasticsearchServiceMock, savedObjectsRepositoryMock } from 'src/core/server/mocks';
98
import {
109
getClusterUuids,
1110
fetchClusterUuids,
1211
handleClusterUuidsResponse,
1312
} from './get_cluster_uuids';
1413

1514
describe('get_cluster_uuids', () => {
16-
const kibanaRequest = undefined;
1715
const callCluster = sinon.stub();
18-
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
19-
const soClient = savedObjectsRepositoryMock.create();
2016
const response = {
2117
aggregations: {
2218
cluster_uuids: {
2319
buckets: [{ key: 'abc' }, { key: 'xyz' }, { key: '123' }],
2420
},
2521
},
2622
};
27-
const expectedUuids = response.aggregations.cluster_uuids.buckets
28-
.map((bucket) => bucket.key)
29-
.map((expectedUuid) => ({ clusterUuid: expectedUuid }));
23+
const expectedUuids = response.aggregations.cluster_uuids.buckets.map((bucket) => bucket.key);
3024
const timestamp = Date.now();
3125

3226
describe('getClusterUuids', () => {
3327
it('returns cluster UUIDs', async () => {
3428
callCluster.withArgs('search').returns(Promise.resolve(response));
35-
expect(
36-
await getClusterUuids(
37-
{ callCluster, esClient, soClient, timestamp, kibanaRequest, usageCollection: {} as any },
38-
{
39-
maxBucketSize: 1,
40-
} as any
41-
)
42-
).toStrictEqual(expectedUuids);
29+
expect(await getClusterUuids(callCluster, timestamp, 1)).toStrictEqual(expectedUuids);
4330
});
4431
});
4532

4633
describe('fetchClusterUuids', () => {
4734
it('searches for clusters', async () => {
4835
callCluster.returns(Promise.resolve(response));
49-
expect(
50-
await fetchClusterUuids(
51-
{ callCluster, esClient, soClient, timestamp, kibanaRequest, usageCollection: {} as any },
52-
{
53-
maxBucketSize: 1,
54-
} as any
55-
)
56-
).toStrictEqual(response);
36+
expect(await fetchClusterUuids(callCluster, timestamp, 1)).toStrictEqual(response);
5737
});
5838
});
5939

‎x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,33 +6,31 @@
66

77
import { get } from 'lodash';
88
import moment from 'moment';
9-
import {
10-
ClusterDetailsGetter,
11-
StatsCollectionConfig,
12-
ClusterDetails,
13-
} from 'src/plugins/telemetry_collection_manager/server';
9+
import { LegacyAPICaller } from 'kibana/server';
1410
import { createQuery } from './create_query';
1511
import {
1612
INDEX_PATTERN_ELASTICSEARCH,
1713
CLUSTER_DETAILS_FETCH_INTERVAL,
1814
} from '../../common/constants';
19-
import { CustomContext } from './get_all_stats';
15+
2016
/**
2117
* Get a list of Cluster UUIDs that exist within the specified timespan.
2218
*/
23-
export const getClusterUuids: ClusterDetailsGetter<CustomContext> = async (
24-
config,
25-
{ maxBucketSize }
26-
) => {
27-
const response = await fetchClusterUuids(config, maxBucketSize);
19+
export async function getClusterUuids(
20+
callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates
21+
timestamp: number,
22+
maxBucketSize: number
23+
) {
24+
const response = await fetchClusterUuids(callCluster, timestamp, maxBucketSize);
2825
return handleClusterUuidsResponse(response);
29-
};
26+
}
3027

3128
/**
3229
* Fetch the aggregated Cluster UUIDs from the monitoring cluster.
3330
*/
3431
export async function fetchClusterUuids(
35-
{ callCluster, timestamp }: StatsCollectionConfig,
32+
callCluster: LegacyAPICaller,
33+
timestamp: number,
3634
maxBucketSize: number
3735
) {
3836
const start = moment(timestamp).subtract(CLUSTER_DETAILS_FETCH_INTERVAL, 'ms').toISOString();
@@ -66,10 +64,7 @@ export async function fetchClusterUuids(
6664
* @param {Object} response The aggregation response
6765
* @return {Array} Strings; each representing a Cluster's UUID.
6866
*/
69-
export function handleClusterUuidsResponse(response: any): ClusterDetails[] {
67+
export function handleClusterUuidsResponse(response: any): string[] {
7068
const uuidBuckets: any[] = get(response, 'aggregations.cluster_uuids.buckets', []);
71-
72-
return uuidBuckets.map((uuidBucket) => ({
73-
clusterUuid: uuidBucket.key as string,
74-
}));
69+
return uuidBuckets.map((uuidBucket) => uuidBucket.key);
7570
}

‎x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.test.ts‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ describe('get_licenses', () => {
1919
},
2020
};
2121
const expectedClusters = response.hits.hits.map((hit) => hit._source);
22-
const clusterUuids = expectedClusters.map((cluster) => ({ clusterUuid: cluster.cluster_uuid }));
22+
const clusterUuids = expectedClusters.map((cluster) => cluster.cluster_uuid);
2323
const expectedLicenses = {
2424
abc: { type: 'basic' },
2525
xyz: { type: 'basic' },
@@ -30,27 +30,15 @@ describe('get_licenses', () => {
3030
it('returns clusters', async () => {
3131
callWith.withArgs('search').returns(Promise.resolve(response));
3232

33-
expect(
34-
await getLicenses(
35-
clusterUuids,
36-
{ callCluster: callWith } as any,
37-
{ maxBucketSize: 1 } as any
38-
)
39-
).toStrictEqual(expectedLicenses);
33+
expect(await getLicenses(clusterUuids, callWith, 1)).toStrictEqual(expectedLicenses);
4034
});
4135
});
4236

4337
describe('fetchLicenses', () => {
4438
it('searches for clusters', async () => {
4539
callWith.returns(response);
4640

47-
expect(
48-
await fetchLicenses(
49-
callWith,
50-
clusterUuids.map(({ clusterUuid }) => clusterUuid),
51-
{ maxBucketSize: 1 } as any
52-
)
53-
).toStrictEqual(response);
41+
expect(await fetchLicenses(callWith, clusterUuids, 1)).toStrictEqual(response);
5442
});
5543
});
5644

0 commit comments

Comments
 (0)