Skip to content

Commit e8d5483

Browse files
dgieselaardgieselaar
committed
Run telemetry task immediately if versions are incompatible; add service environments telemetry
1 parent eceb500 commit e8d5483

8 files changed

Lines changed: 221 additions & 39 deletions

File tree

‎x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap‎

Lines changed: 28 additions & 26 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/plugins/apm/common/apm_telemetry.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ export function getApmTelemetryMapping() {
7878
properties: {
7979
expected_metric_document_count: long,
8080
transaction_count: long,
81+
ratio: long,
8182
},
8283
};
8384

@@ -102,12 +103,14 @@ export function getApmTelemetryMapping() {
102103
properties: {
103104
current_implementation: aggregatedTransactionsProperties,
104105
no_observer_name: aggregatedTransactionsProperties,
105-
no_rum: aggregatedTransactionsProperties,
106-
no_rum_no_observer_name: aggregatedTransactionsProperties,
107-
only_rum: aggregatedTransactionsProperties,
108-
only_rum_no_observer_name: aggregatedTransactionsProperties,
106+
with_country: aggregatedTransactionsProperties,
109107
},
110108
},
109+
environments: {
110+
services_without_environment: long,
111+
services_with_multiple_environments: long,
112+
top_enviroments: keyword,
113+
},
111114
cloud: {
112115
properties: {
113116
availability_zone: keyword,
@@ -227,6 +230,7 @@ export function getApmTelemetryMapping() {
227230
agents: tookProperties,
228231
cardinality: tookProperties,
229232
cloud: tookProperties,
233+
environments: tookProperties,
230234
groupings: tookProperties,
231235
indices_stats: tookProperties,
232236
integrations: tookProperties,

‎x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66

77
import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
88
import { tasks } from './tasks';
9+
import {
10+
SERVICE_NAME,
11+
SERVICE_ENVIRONMENT,
12+
} from '../../../../common/elasticsearch_fieldnames';
913

1014
describe('data telemetry collection tasks', () => {
1115
const indices = {
@@ -17,6 +21,63 @@ describe('data telemetry collection tasks', () => {
1721
/* eslint-enable @typescript-eslint/naming-convention */
1822
} as ApmIndicesConfig;
1923

24+
describe('environments', () => {
25+
const task = tasks.find((t) => t.name === 'environments');
26+
27+
it('returns environment information', async () => {
28+
const search = jest.fn().mockResolvedValueOnce({
29+
aggregations: {
30+
environments: {
31+
buckets: [
32+
{
33+
key: 'production',
34+
},
35+
{
36+
key: 'testing',
37+
},
38+
],
39+
},
40+
service_environments: {
41+
buckets: [
42+
{
43+
key: {
44+
[SERVICE_NAME]: 'opbeans-node',
45+
[SERVICE_ENVIRONMENT]: 'production',
46+
},
47+
},
48+
{
49+
key: {
50+
[SERVICE_NAME]: 'opbeans-node',
51+
[SERVICE_ENVIRONMENT]: null,
52+
},
53+
},
54+
{
55+
key: {
56+
[SERVICE_NAME]: 'opbeans-java',
57+
[SERVICE_ENVIRONMENT]: 'production',
58+
},
59+
},
60+
{
61+
key: {
62+
[SERVICE_NAME]: 'opbeans-rum',
63+
[SERVICE_ENVIRONMENT]: null,
64+
},
65+
},
66+
],
67+
},
68+
},
69+
});
70+
71+
expect(await task?.executor({ search, indices } as any)).toEqual({
72+
environments: {
73+
services_with_multiple_environments: 1,
74+
services_without_environment: 2,
75+
top_environments: ['production', 'testing'],
76+
},
77+
});
78+
});
79+
});
80+
2081
describe('aggregated_transactions', () => {
2182
const task = tasks.find((t) => t.name === 'aggregated_transactions');
2283

‎x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66
import { ValuesType } from 'utility-types';
7-
import { flatten, merge, sortBy, sum } from 'lodash';
7+
import { flatten, merge, sortBy, sum, pickBy } from 'lodash';
88
import { AggregationOptionsByType } from '../../../../typings/elasticsearch/aggregations';
99
import { ProcessorEvent } from '../../../../common/processor_event';
1010
import { TelemetryTask } from '.';
@@ -294,6 +294,87 @@ export const tasks: TelemetryTask[] = [
294294
return { cloud };
295295
},
296296
},
297+
{
298+
name: 'environments',
299+
executor: async ({ indices, search }) => {
300+
const response = await search({
301+
index: [indices['apm_oss.transactionIndices']],
302+
body: {
303+
query: {
304+
bool: {
305+
filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }],
306+
},
307+
},
308+
aggs: {
309+
environments: {
310+
terms: {
311+
field: SERVICE_ENVIRONMENT,
312+
size: 5,
313+
},
314+
},
315+
service_environments: {
316+
composite: {
317+
size: 1000,
318+
sources: [
319+
{
320+
[SERVICE_ENVIRONMENT]: {
321+
terms: {
322+
field: SERVICE_ENVIRONMENT,
323+
missing_bucket: true,
324+
},
325+
},
326+
},
327+
{
328+
[SERVICE_NAME]: {
329+
terms: {
330+
field: SERVICE_NAME,
331+
},
332+
},
333+
},
334+
],
335+
},
336+
},
337+
},
338+
},
339+
});
340+
341+
const topEnvironments =
342+
response.aggregations?.environments.buckets.map(
343+
(bucket) => bucket.key
344+
) ?? [];
345+
const serviceEnvironments: Record<string, Array<string | null>> = {};
346+
347+
const buckets = response.aggregations?.service_environments.buckets ?? [];
348+
349+
buckets.forEach((bucket) => {
350+
const serviceName = bucket.key['service.name'];
351+
const environment = bucket.key['service.environment'] as string | null;
352+
353+
const environments = serviceEnvironments[serviceName] ?? [];
354+
355+
serviceEnvironments[serviceName] = environments.concat(environment);
356+
});
357+
358+
const servicesWithoutEnvironment = Object.keys(
359+
pickBy(serviceEnvironments, (environments) =>
360+
environments.includes(null)
361+
)
362+
);
363+
364+
const servicesWithMultipleEnvironments = Object.keys(
365+
pickBy(serviceEnvironments, (environments) => environments.length > 1)
366+
);
367+
368+
return {
369+
environments: {
370+
services_without_environment: servicesWithoutEnvironment.length,
371+
services_with_multiple_environments:
372+
servicesWithMultipleEnvironments.length,
373+
top_environments: topEnvironments as string[],
374+
},
375+
};
376+
},
377+
},
297378
{
298379
name: 'processor_events',
299380
executor: async ({ indices, search }) => {

‎x-pack/plugins/apm/server/lib/apm_telemetry/index.ts‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import { Observable } from 'rxjs';
77
import { take } from 'rxjs/operators';
88
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
9+
import { DeepRequired } from 'utility-types';
910
import {
1011
CoreSetup,
1112
Logger,
@@ -27,6 +28,7 @@ import {
2728
collectDataTelemetry,
2829
CollectTelemetryParams,
2930
} from './collect_data_telemetry';
31+
import { APMDataTelemetry } from './types';
3032

3133
const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task';
3234

@@ -36,12 +38,14 @@ export async function createApmTelemetry({
3638
usageCollector,
3739
taskManager,
3840
logger,
41+
kibanaVersion,
3942
}: {
4043
core: CoreSetup;
4144
config$: Observable<APMConfig>;
4245
usageCollector: UsageCollectionSetup;
4346
taskManager: TaskManagerSetupContract;
4447
logger: Logger;
48+
kibanaVersion: string;
4549
}) {
4650
taskManager.registerTaskDefinitions({
4751
[APM_TELEMETRY_TASK_NAME]: {
@@ -95,7 +99,10 @@ export async function createApmTelemetry({
9599

96100
await savedObjectsClient.create(
97101
APM_TELEMETRY_SAVED_OBJECT_TYPE,
98-
dataTelemetry,
102+
{
103+
...dataTelemetry,
104+
kibanaVersion,
105+
},
99106
{ id: APM_TELEMETRY_SAVED_OBJECT_TYPE, overwrite: true }
100107
);
101108
};
@@ -105,12 +112,14 @@ export async function createApmTelemetry({
105112
schema: getApmTelemetryMapping(),
106113
fetch: async () => {
107114
try {
108-
const data = (
115+
const { kibanaVersion: storedKibanaVersion, ...data } = (
109116
await savedObjectsClient.get(
110117
APM_TELEMETRY_SAVED_OBJECT_TYPE,
111118
APM_TELEMETRY_SAVED_OBJECT_ID
112119
)
113-
).attributes;
120+
).attributes as { kibanaVersion: string } & DeepRequired<
121+
APMDataTelemetry
122+
>;
114123

115124
return data;
116125
} catch (err) {
@@ -126,7 +135,7 @@ export async function createApmTelemetry({
126135

127136
usageCollector.registerCollector(collector);
128137

129-
core.getStartServices().then(([_coreStart, pluginsStart]) => {
138+
core.getStartServices().then(async ([_coreStart, pluginsStart]) => {
130139
const { taskManager: taskManagerStart } = pluginsStart as {
131140
taskManager: TaskManagerStartContract;
132141
};
@@ -141,5 +150,25 @@ export async function createApmTelemetry({
141150
params: {},
142151
state: {},
143152
});
153+
154+
try {
155+
const currentData = (
156+
await savedObjectsClient.get(
157+
APM_TELEMETRY_SAVED_OBJECT_TYPE,
158+
APM_TELEMETRY_SAVED_OBJECT_ID
159+
)
160+
).attributes as { kibanaVersion?: string };
161+
162+
if (currentData.kibanaVersion !== kibanaVersion) {
163+
logger.debug(
164+
`Stored telemetry is out of date. Task will run immediately. Stored: ${currentData.kibanaVersion}, expected: ${kibanaVersion}`
165+
);
166+
taskManagerStart.runNow(APM_TELEMETRY_TASK_NAME);
167+
}
168+
} catch (err) {
169+
if (!SavedObjectsErrorHelpers.isNotFoundError(err)) {
170+
logger.warn('Failed to fetch saved telemetry data.');
171+
}
172+
}
144173
});
145174
}

0 commit comments

Comments
 (0)