Skip to content

Commit a448d23

Browse files
afharoafharo
committed
[Telemetry] Move Monitoring collection strategy to a collector
1 parent 0faf8c2 commit a448d23

17 files changed

Lines changed: 350 additions & 267 deletions

File tree

‎src/plugins/telemetry_collection_manager/server/plugin.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,9 +286,9 @@ export class TelemetryCollectionManagerPlugin
286286
return stats.map((stat) => {
287287
const license = licenses[stat.cluster_uuid];
288288
return {
289+
collectionSource: collection.title,
289290
...(license ? { license } : {}),
290291
...stat,
291-
collectionSource: collection.title,
292292
};
293293
});
294294
}

‎x-pack/plugins/monitoring/kibana.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
],
1414
"optionalPlugins": [
1515
"infra",
16-
"telemetryCollectionManager",
1716
"usageCollection",
1817
"home",
1918
"cloud",

‎x-pack/plugins/monitoring/server/plugin.ts‎

Lines changed: 7 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ import {
2020
CoreStart,
2121
CustomHttpResponseOptions,
2222
ResponseError,
23-
IClusterClient,
24-
SavedObjectsServiceStart,
2523
} from 'kibana/server';
2624
import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
2725
import {
@@ -41,7 +39,7 @@ import { initInfraSource } from './lib/logs/init_infra_source';
4139
import { mbSafeQuery } from './lib/mb_safe_query';
4240
import { instantiateClient } from './es_client/instantiate_client';
4341
import { registerCollectors } from './kibana_monitoring/collectors';
44-
import { registerMonitoringCollection } from './telemetry_collection';
42+
import { registerMonitoringTelemetryCollection } from './telemetry_collection';
4543
import { LicenseService } from './license_service';
4644
import { AlertsFactory } from './alerts';
4745
import {
@@ -76,8 +74,6 @@ export class Plugin {
7674
private monitoringCore = {} as MonitoringCore;
7775
private legacyShimDependencies = {} as LegacyShimDependencies;
7876
private bulkUploader: IBulkUploader = {} as IBulkUploader;
79-
private telemetryElasticsearchClient: IClusterClient | undefined;
80-
private telemetrySavedObjectsService: SavedObjectsServiceStart | undefined;
8177

8278
constructor(initializerContext: PluginInitializerContext) {
8379
this.initializerContext = initializerContext;
@@ -145,19 +141,6 @@ export class Plugin {
145141
plugins.alerts?.registerType(alert.getAlertType());
146142
}
147143

148-
// Initialize telemetry
149-
if (plugins.telemetryCollectionManager) {
150-
registerMonitoringCollection({
151-
telemetryCollectionManager: plugins.telemetryCollectionManager,
152-
esCluster: this.cluster,
153-
esClientGetter: () => this.telemetryElasticsearchClient,
154-
soServiceGetter: () => this.telemetrySavedObjectsService,
155-
customContext: {
156-
maxBucketSize: config.ui.max_bucket_size,
157-
},
158-
});
159-
}
160-
161144
// Register collector objects for stats to show up in the APIs
162145
if (plugins.usageCollection) {
163146
core.savedObjects.registerType({
@@ -174,6 +157,11 @@ export class Plugin {
174157
});
175158

176159
registerCollectors(plugins.usageCollection, config, cluster.callAsInternalUser);
160+
registerMonitoringTelemetryCollection(
161+
plugins.usageCollection,
162+
cluster.callAsInternalUser,
163+
config.ui.max_bucket_size
164+
);
177165
}
178166

179167
// Always create the bulk uploader
@@ -253,16 +241,7 @@ export class Plugin {
253241
};
254242
}
255243

256-
start({ elasticsearch, savedObjects }: CoreStart) {
257-
// TODO: For the telemetry plugin to work, we need to provide the new ES client.
258-
// The new client should be inititalized with a similar config to `this.cluster` but, since we're not using
259-
// the new client in Monitoring Telemetry collection yet, setting the local client allows progress for now.
260-
// The usage collector `fetch` method has been refactored to accept a `collectorFetchContext` object,
261-
// exposing both es clients and the saved objects client.
262-
// We will update the client in a follow up PR.
263-
this.telemetryElasticsearchClient = elasticsearch.client;
264-
this.telemetrySavedObjectsService = savedObjects;
265-
}
244+
start() {}
266245

267246
stop() {
268247
if (this.cluster) {

‎x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts‎

Lines changed: 2 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,10 @@ import { getStackStats, getAllStats, handleAllStats } from './get_all_stats';
99
import { ESClusterStats } from './get_es_stats';
1010
import { KibanaStats } from './get_kibana_stats';
1111
import { ClustersHighLevelStats } from './get_high_level_stats';
12-
import { coreMock } from 'src/core/server/mocks';
1312

1413
describe('get_all_stats', () => {
1514
const timestamp = Date.now();
1615
const callCluster = sinon.stub();
17-
const esClient = sinon.stub();
18-
const soClient = sinon.stub();
1916

2017
const esClusters = [
2118
{ cluster_uuid: 'a' },
@@ -172,23 +169,7 @@ describe('get_all_stats', () => {
172169
.onCall(4)
173170
.returns(Promise.resolve({})); // Beats state
174171

175-
expect(
176-
await getAllStats(
177-
[{ clusterUuid: 'a' }],
178-
{
179-
callCluster: callCluster as any,
180-
esClient: esClient as any,
181-
soClient: soClient as any,
182-
usageCollection: {} as any,
183-
timestamp,
184-
},
185-
{
186-
logger: coreMock.createPluginInitializerContext().logger.get('test'),
187-
version: 'version',
188-
maxBucketSize: 1,
189-
}
190-
)
191-
).toStrictEqual(allClusters);
172+
expect(await getAllStats(['a'], callCluster, timestamp, 1)).toStrictEqual(allClusters);
192173
});
193174

194175
it('returns empty clusters', async () => {
@@ -198,23 +179,7 @@ describe('get_all_stats', () => {
198179

199180
callCluster.withArgs('search').returns(Promise.resolve(clusterUuidsResponse));
200181

201-
expect(
202-
await getAllStats(
203-
[],
204-
{
205-
callCluster: callCluster as any,
206-
esClient: esClient as any,
207-
soClient: soClient as any,
208-
usageCollection: {} as any,
209-
timestamp,
210-
},
211-
{
212-
logger: coreMock.createPluginInitializerContext().logger.get('test'),
213-
version: 'version',
214-
maxBucketSize: 1,
215-
}
216-
)
217-
).toStrictEqual([]);
182+
expect(await getAllStats([], callCluster, timestamp, 1)).toStrictEqual([]);
218183
});
219184
});
220185

‎x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
import { set } from '@elastic/safer-lodash-set';
88
import { get, merge } from 'lodash';
99

10-
import { StatsGetter } from 'src/plugins/telemetry_collection_manager/server';
1110
import moment from 'moment';
11+
import { LegacyAPICaller } from 'kibana/server';
1212
import {
1313
LOGSTASH_SYSTEM_ID,
1414
KIBANA_SYSTEM_ID,
@@ -20,24 +20,20 @@ import { getKibanaStats, KibanaStats } from './get_kibana_stats';
2020
import { getBeatsStats, BeatsStatsByClusterUuid } from './get_beats_stats';
2121
import { getHighLevelStats, ClustersHighLevelStats } from './get_high_level_stats';
2222

23-
export interface CustomContext {
24-
maxBucketSize: number;
25-
}
2623
/**
2724
* Get statistics for all products joined by Elasticsearch cluster.
2825
* Returns the array of clusters joined with the Kibana and Logstash instances.
2926
*
3027
*/
31-
export const getAllStats: StatsGetter<CustomContext> = async (
32-
clustersDetails,
33-
{ callCluster, timestamp },
34-
{ maxBucketSize }
35-
) => {
28+
export async function getAllStats(
29+
clusterUuids: string[],
30+
callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates
31+
timestamp: number,
32+
maxBucketSize: number
33+
) {
3634
const start = moment(timestamp).subtract(USAGE_FETCH_INTERVAL, 'ms').toISOString();
3735
const end = moment(timestamp).toISOString();
3836

39-
const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid);
40-
4137
const [esClusters, kibana, logstash, beats] = await Promise.all([
4238
getElasticsearchStats(callCluster, clusterUuids, maxBucketSize), // cluster_stats, stack_stats.xpack, cluster_name/uuid, license, version
4339
getKibanaStats(callCluster, clusterUuids, start, end, maxBucketSize), // stack_stats.kibana
@@ -46,7 +42,7 @@ export const getAllStats: StatsGetter<CustomContext> = async (
4642
]);
4743

4844
return handleAllStats(esClusters, { kibana, logstash, beats });
49-
};
45+
}
5046

5147
/**
5248
* Combine the statistics from the stack to create "cluster" stats that associate all products together based on the cluster

‎x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts‎

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
*/
66

77
import sinon from 'sinon';
8-
import { elasticsearchServiceMock, savedObjectsRepositoryMock } from 'src/core/server/mocks';
98
import {
109
getClusterUuids,
1110
fetchClusterUuids,
@@ -14,45 +13,27 @@ import {
1413

1514
describe('get_cluster_uuids', () => {
1615
const callCluster = sinon.stub();
17-
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
18-
const soClient = savedObjectsRepositoryMock.create();
1916
const response = {
2017
aggregations: {
2118
cluster_uuids: {
2219
buckets: [{ key: 'abc' }, { key: 'xyz' }, { key: '123' }],
2320
},
2421
},
2522
};
26-
const expectedUuids = response.aggregations.cluster_uuids.buckets
27-
.map((bucket) => bucket.key)
28-
.map((expectedUuid) => ({ clusterUuid: expectedUuid }));
23+
const expectedUuids = response.aggregations.cluster_uuids.buckets.map((bucket) => bucket.key);
2924
const timestamp = Date.now();
3025

3126
describe('getClusterUuids', () => {
3227
it('returns cluster UUIDs', async () => {
3328
callCluster.withArgs('search').returns(Promise.resolve(response));
34-
expect(
35-
await getClusterUuids(
36-
{ callCluster, esClient, soClient, timestamp, usageCollection: {} as any },
37-
{
38-
maxBucketSize: 1,
39-
} as any
40-
)
41-
).toStrictEqual(expectedUuids);
29+
expect(await getClusterUuids(callCluster, timestamp, 1)).toStrictEqual(expectedUuids);
4230
});
4331
});
4432

4533
describe('fetchClusterUuids', () => {
4634
it('searches for clusters', async () => {
4735
callCluster.returns(Promise.resolve(response));
48-
expect(
49-
await fetchClusterUuids(
50-
{ callCluster, esClient, soClient, timestamp, usageCollection: {} as any },
51-
{
52-
maxBucketSize: 1,
53-
} as any
54-
)
55-
).toStrictEqual(response);
36+
expect(await fetchClusterUuids(callCluster, timestamp, 1)).toStrictEqual(response);
5637
});
5738
});
5839

‎x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,33 +6,31 @@
66

77
import { get } from 'lodash';
88
import moment from 'moment';
9-
import {
10-
ClusterDetailsGetter,
11-
StatsCollectionConfig,
12-
ClusterDetails,
13-
} from 'src/plugins/telemetry_collection_manager/server';
9+
import { LegacyAPICaller } from 'kibana/server';
1410
import { createQuery } from './create_query';
1511
import {
1612
INDEX_PATTERN_ELASTICSEARCH,
1713
CLUSTER_DETAILS_FETCH_INTERVAL,
1814
} from '../../common/constants';
19-
import { CustomContext } from './get_all_stats';
15+
2016
/**
2117
* Get a list of Cluster UUIDs that exist within the specified timespan.
2218
*/
23-
export const getClusterUuids: ClusterDetailsGetter<CustomContext> = async (
24-
config,
25-
{ maxBucketSize }
26-
) => {
27-
const response = await fetchClusterUuids(config, maxBucketSize);
19+
export async function getClusterUuids(
20+
callCluster: LegacyAPICaller, // TODO: To be changed to the new ES client when the plugin migrates
21+
timestamp: number,
22+
maxBucketSize: number
23+
) {
24+
const response = await fetchClusterUuids(callCluster, timestamp, maxBucketSize);
2825
return handleClusterUuidsResponse(response);
29-
};
26+
}
3027

3128
/**
3229
* Fetch the aggregated Cluster UUIDs from the monitoring cluster.
3330
*/
3431
export async function fetchClusterUuids(
35-
{ callCluster, timestamp }: StatsCollectionConfig,
32+
callCluster: LegacyAPICaller,
33+
timestamp: number,
3634
maxBucketSize: number
3735
) {
3836
const start = moment(timestamp).subtract(CLUSTER_DETAILS_FETCH_INTERVAL, 'ms').toISOString();
@@ -66,10 +64,7 @@ export async function fetchClusterUuids(
6664
* @param {Object} response The aggregation response
6765
* @return {Array} Strings; each representing a Cluster's UUID.
6866
*/
69-
export function handleClusterUuidsResponse(response: any): ClusterDetails[] {
67+
export function handleClusterUuidsResponse(response: any): string[] {
7068
const uuidBuckets: any[] = get(response, 'aggregations.cluster_uuids.buckets', []);
71-
72-
return uuidBuckets.map((uuidBucket) => ({
73-
clusterUuid: uuidBucket.key as string,
74-
}));
69+
return uuidBuckets.map((uuidBucket) => uuidBucket.key);
7570
}

‎x-pack/plugins/monitoring/server/telemetry_collection/get_licenses.test.ts‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ describe('get_licenses', () => {
1919
},
2020
};
2121
const expectedClusters = response.hits.hits.map((hit) => hit._source);
22-
const clusterUuids = expectedClusters.map((cluster) => ({ clusterUuid: cluster.cluster_uuid }));
22+
const clusterUuids = expectedClusters.map((cluster) => cluster.cluster_uuid);
2323
const expectedLicenses = {
2424
abc: { type: 'basic' },
2525
xyz: { type: 'basic' },
@@ -30,27 +30,15 @@ describe('get_licenses', () => {
3030
it('returns clusters', async () => {
3131
callWith.withArgs('search').returns(Promise.resolve(response));
3232

33-
expect(
34-
await getLicenses(
35-
clusterUuids,
36-
{ callCluster: callWith } as any,
37-
{ maxBucketSize: 1 } as any
38-
)
39-
).toStrictEqual(expectedLicenses);
33+
expect(await getLicenses(clusterUuids, callWith, 1)).toStrictEqual(expectedLicenses);
4034
});
4135
});
4236

4337
describe('fetchLicenses', () => {
4438
it('searches for clusters', async () => {
4539
callWith.returns(response);
4640

47-
expect(
48-
await fetchLicenses(
49-
callWith,
50-
clusterUuids.map(({ clusterUuid }) => clusterUuid),
51-
{ maxBucketSize: 1 } as any
52-
)
53-
).toStrictEqual(response);
41+
expect(await fetchLicenses(callWith, clusterUuids, 1)).toStrictEqual(response);
5442
});
5543
});
5644

0 commit comments

Comments
 (0)