Skip to content

Commit dbfc8f7

Browse files
weltenwortweltenwort
committed
[Logs UI] Generalize ML module management (#50662)
This abstracts the specific job details out of the ML module management hooks to enable re-use with the upcoming categorization module. closes #50322
1 parent 85f55bc commit dbfc8f7

61 files changed

Lines changed: 910 additions & 704 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export type GetLogEntryRateRequestPayload = rt.TypeOf<typeof getLogEntryRateRequ
2929
* response
3030
*/
3131

32-
export const logEntryRateAnomaly = rt.type({
32+
export const logEntryRateAnomalyRT = rt.type({
3333
actualLogEntryRate: rt.number,
3434
anomalyScore: rt.number,
3535
duration: rt.number,
@@ -39,22 +39,26 @@ export const logEntryRateAnomaly = rt.type({
3939

4040
export const logEntryRatePartitionRT = rt.type({
4141
analysisBucketCount: rt.number,
42-
anomalies: rt.array(logEntryRateAnomaly),
42+
anomalies: rt.array(logEntryRateAnomalyRT),
4343
averageActualLogEntryRate: rt.number,
4444
maximumAnomalyScore: rt.number,
4545
numberOfLogEntries: rt.number,
4646
partitionId: rt.string,
4747
});
4848

49-
export const logEntryRateHistogramBucket = rt.type({
49+
export type LogEntryRatePartition = rt.TypeOf<typeof logEntryRatePartitionRT>;
50+
51+
export const logEntryRateHistogramBucketRT = rt.type({
5052
partitions: rt.array(logEntryRatePartitionRT),
5153
startTime: rt.number,
5254
});
5355

56+
export type LogEntryRateHistogramBucket = rt.TypeOf<typeof logEntryRateHistogramBucketRT>;
57+
5458
export const getLogEntryRateSuccessReponsePayloadRT = rt.type({
5559
data: rt.type({
5660
bucketDuration: rt.number,
57-
histogramBuckets: rt.array(logEntryRateHistogramBucket),
61+
histogramBuckets: rt.array(logEntryRateHistogramBucketRT),
5862
totalNumberOfLogEntries: rt.number,
5963
}),
6064
});

‎x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
export * from './indices';
7+
export * from './log_entry_rate_indices';

x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/indices.ts renamed to x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/log_entry_rate_indices.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,24 @@
66

77
import * as rt from 'io-ts';
88

9-
export const LOG_ANALYSIS_VALIDATION_INDICES_PATH = '/api/infra/log_analysis/validation/indices';
9+
export const LOG_ANALYSIS_VALIDATE_INDICES_PATH =
10+
'/api/infra/log_analysis/validation/log_entry_rate_indices';
1011

1112
/**
1213
* Request types
1314
*/
15+
export const validationIndicesFieldSpecificationRT = rt.type({
16+
name: rt.string,
17+
validTypes: rt.array(rt.string),
18+
});
19+
20+
export type ValidationIndicesFieldSpecification = rt.TypeOf<
21+
typeof validationIndicesFieldSpecificationRT
22+
>;
23+
1424
export const validationIndicesRequestPayloadRT = rt.type({
1525
data: rt.type({
16-
timestampField: rt.string,
26+
fields: rt.array(validationIndicesFieldSpecificationRT),
1727
indices: rt.array(rt.string),
1828
}),
1929
});

‎x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,25 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import { JobType } from './log_analysis';
7+
import * as rt from 'io-ts';
88

99
export const bucketSpan = 900000;
1010

11+
export const partitionField = 'event.dataset';
12+
1113
export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
1214
`kibana-logs-ui-${spaceId}-${sourceId}-`;
1315

14-
export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) =>
16+
export const getJobId = (spaceId: string, sourceId: string, jobType: string) =>
1517
`${getJobIdPrefix(spaceId, sourceId)}${jobType}`;
1618

17-
export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) =>
19+
export const getDatafeedId = (spaceId: string, sourceId: string, jobType: string) =>
1820
`datafeed-${getJobId(spaceId, sourceId, jobType)}`;
1921

20-
export const getAllModuleJobIds = (spaceId: string, sourceId: string) => [
21-
getJobId(spaceId, sourceId, 'log-entry-rate'),
22-
];
22+
export const jobSourceConfigurationRT = rt.type({
23+
indexPattern: rt.string,
24+
timestampField: rt.string,
25+
bucketSpan: rt.number,
26+
});
27+
28+
export type JobSourceConfiguration = rt.TypeOf<typeof jobSourceConfigurationRT>;

x-pack/legacy/plugins/infra/public/pages/logs/analysis/sections/analyze_in_ml_button.tsx renamed to x-pack/legacy/plugins/infra/public/components/logging/log_analysis_results/analyze_in_ml_button.tsx

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,15 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
import React from 'react';
8-
import url from 'url';
97
import { EuiButton } from '@elastic/eui';
108
import { FormattedMessage } from '@kbn/i18n/react';
9+
import React from 'react';
10+
import { encode } from 'rison-node';
1111
import chrome from 'ui/chrome';
1212
import { QueryString } from 'ui/utils/query_string';
13-
import { encode } from 'rison-node';
14-
import { TimeRange } from '../../../../../common/http_api/shared/time_range';
13+
import url from 'url';
14+
15+
import { TimeRange } from '../../../../common/http_api/shared/time_range';
1516

1617
export const AnalyzeInMlButton: React.FunctionComponent<{
1718
jobId: string;

‎x-pack/legacy/plugins/infra/public/components/logging/log_analysis_results/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
export * from './analyze_in_ml_button';

‎x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_api_types.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,30 @@
66

77
import * as rt from 'io-ts';
88

9+
import { jobSourceConfigurationRT } from '../../../../../common/log_analysis';
10+
911
export const jobCustomSettingsRT = rt.partial({
1012
job_revision: rt.number,
11-
logs_source_config: rt.partial({
12-
indexPattern: rt.string,
13-
timestampField: rt.string,
14-
bucketSpan: rt.number,
13+
logs_source_config: rt.partial(jobSourceConfigurationRT.props),
14+
});
15+
16+
export const getMlCapabilitiesResponsePayloadRT = rt.type({
17+
capabilities: rt.type({
18+
canGetJobs: rt.boolean,
19+
canCreateJob: rt.boolean,
20+
canDeleteJob: rt.boolean,
21+
canOpenJob: rt.boolean,
22+
canCloseJob: rt.boolean,
23+
canForecastJob: rt.boolean,
24+
canGetDatafeeds: rt.boolean,
25+
canStartStopDatafeed: rt.boolean,
26+
canUpdateJob: rt.boolean,
27+
canUpdateDatafeed: rt.boolean,
28+
canPreviewDatafeed: rt.boolean,
1529
}),
30+
isPlatinumOrTrialLicense: rt.boolean,
31+
mlFeatureEnabledInSpace: rt.boolean,
32+
upgradeInProgress: rt.boolean,
1633
});
34+
35+
export type GetMlCapabilitiesResponsePayload = rt.TypeOf<typeof getMlCapabilitiesResponsePayloadRT>;

‎x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,22 @@ import { pipe } from 'fp-ts/lib/pipeable';
99
import { fold } from 'fp-ts/lib/Either';
1010
import { identity } from 'fp-ts/lib/function';
1111
import { kfetch } from 'ui/kfetch';
12-
import { getAllModuleJobIds, getDatafeedId } from '../../../../../common/log_analysis';
12+
13+
import { getDatafeedId, getJobId } from '../../../../../common/log_analysis';
1314
import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
1415

15-
export const callDeleteJobs = async (spaceId: string, sourceId: string) => {
16+
export const callDeleteJobs = async <JobType extends string>(
17+
spaceId: string,
18+
sourceId: string,
19+
jobTypes: JobType[]
20+
) => {
1621
// NOTE: Deleting the jobs via this API will delete the datafeeds at the same time
1722
const deleteJobsResponse = await kfetch({
1823
method: 'POST',
1924
pathname: '/api/ml/jobs/delete_jobs',
2025
body: JSON.stringify(
2126
deleteJobsRequestPayloadRT.encode({
22-
jobIds: getAllModuleJobIds(spaceId, sourceId),
27+
jobIds: jobTypes.map(jobType => getJobId(spaceId, sourceId, jobType)),
2328
})
2429
),
2530
});
@@ -42,15 +47,24 @@ export const callGetJobDeletionTasks = async () => {
4247
);
4348
};
4449

45-
export const callStopDatafeed = async (spaceId: string, sourceId: string) => {
50+
export const callStopDatafeeds = async <JobType extends string>(
51+
spaceId: string,
52+
sourceId: string,
53+
jobTypes: JobType[]
54+
) => {
4655
// Stop datafeed due to https://github.com/elastic/kibana/issues/44652
4756
const stopDatafeedResponse = await kfetch({
4857
method: 'POST',
49-
pathname: `/api/ml/datafeeds/${getDatafeedId(spaceId, sourceId, 'log-entry-rate')}/_stop`,
58+
pathname: '/api/ml/jobs/stop_datafeeds',
59+
body: JSON.stringify(
60+
stopDatafeedsRequestPayloadRT.encode({
61+
datafeedIds: jobTypes.map(jobType => getDatafeedId(spaceId, sourceId, jobType)),
62+
})
63+
),
5064
});
5165

5266
return pipe(
53-
stopDatafeedResponsePayloadRT.decode(stopDatafeedResponse),
67+
stopDatafeedsResponsePayloadRT.decode(stopDatafeedResponse),
5468
fold(throwErrors(createPlainError), identity)
5569
);
5670
};
@@ -68,10 +82,19 @@ export const deleteJobsResponsePayloadRT = rt.record(
6882
})
6983
);
7084

85+
export type DeleteJobsResponsePayload = rt.TypeOf<typeof deleteJobsResponsePayloadRT>;
86+
7187
export const getJobDeletionTasksResponsePayloadRT = rt.type({
7288
jobIds: rt.array(rt.string),
7389
});
7490

75-
export const stopDatafeedResponsePayloadRT = rt.type({
76-
stopped: rt.boolean,
91+
export const stopDatafeedsRequestPayloadRT = rt.type({
92+
datafeedIds: rt.array(rt.string),
7793
});
94+
95+
export const stopDatafeedsResponsePayloadRT = rt.record(
96+
rt.string,
97+
rt.type({
98+
stopped: rt.boolean,
99+
})
100+
);

‎x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,19 @@ import { kfetch } from 'ui/kfetch';
1212

1313
import { jobCustomSettingsRT } from './ml_api_types';
1414
import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
15-
import { getAllModuleJobIds } from '../../../../../common/log_analysis';
15+
import { getJobId } from '../../../../../common/log_analysis';
1616

17-
export const callJobsSummaryAPI = async (spaceId: string, sourceId: string) => {
17+
export const callJobsSummaryAPI = async <JobType extends string>(
18+
spaceId: string,
19+
sourceId: string,
20+
jobTypes: JobType[]
21+
) => {
1822
const response = await kfetch({
1923
method: 'POST',
2024
pathname: '/api/ml/jobs/jobs_summary',
2125
body: JSON.stringify(
2226
fetchJobStatusRequestPayloadRT.encode({
23-
jobIds: getAllModuleJobIds(spaceId, sourceId),
27+
jobIds: jobTypes.map(jobType => getJobId(spaceId, sourceId, jobType)),
2428
})
2529
),
2630
});

‎x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts‎

Lines changed: 13 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import { kfetch } from 'ui/kfetch';
1212

1313
import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
1414
import { getJobIdPrefix } from '../../../../../common/log_analysis';
15-
import { jobCustomSettingsRT } from './ml_api_types';
1615

1716
export const callSetupMlModuleAPI = async (
1817
moduleId: string,
@@ -21,8 +20,8 @@ export const callSetupMlModuleAPI = async (
2120
spaceId: string,
2221
sourceId: string,
2322
indexPattern: string,
24-
timeField: string,
25-
bucketSpan: number
23+
jobOverrides: SetupMlModuleJobOverrides[] = [],
24+
datafeedOverrides: SetupMlModuleDatafeedOverrides[] = []
2625
) => {
2726
const response = await kfetch({
2827
method: 'POST',
@@ -34,25 +33,8 @@ export const callSetupMlModuleAPI = async (
3433
indexPatternName: indexPattern,
3534
prefix: getJobIdPrefix(spaceId, sourceId),
3635
startDatafeed: true,
37-
jobOverrides: [
38-
{
39-
job_id: 'log-entry-rate' as const,
40-
analysis_config: {
41-
bucket_span: `${bucketSpan}ms`,
42-
},
43-
data_description: {
44-
time_field: timeField,
45-
},
46-
custom_settings: {
47-
logs_source_config: {
48-
indexPattern,
49-
timestampField: timeField,
50-
bucketSpan,
51-
},
52-
},
53-
},
54-
],
55-
datafeedOverrides: [],
36+
jobOverrides,
37+
datafeedOverrides,
5638
})
5739
),
5840
});
@@ -68,23 +50,20 @@ const setupMlModuleTimeParamsRT = rt.partial({
6850
end: rt.number,
6951
});
7052

71-
const setupMlModuleLogEntryRateJobOverridesRT = rt.type({
72-
job_id: rt.literal('log-entry-rate'),
73-
analysis_config: rt.type({
74-
bucket_span: rt.string,
75-
}),
76-
data_description: rt.type({
77-
time_field: rt.string,
78-
}),
79-
custom_settings: jobCustomSettingsRT,
80-
});
53+
const setupMlModuleJobOverridesRT = rt.object;
54+
55+
export type SetupMlModuleJobOverrides = rt.TypeOf<typeof setupMlModuleJobOverridesRT>;
56+
57+
const setupMlModuleDatafeedOverridesRT = rt.object;
58+
59+
export type SetupMlModuleDatafeedOverrides = rt.TypeOf<typeof setupMlModuleDatafeedOverridesRT>;
8160

8261
const setupMlModuleRequestParamsRT = rt.type({
8362
indexPatternName: rt.string,
8463
prefix: rt.string,
8564
startDatafeed: rt.boolean,
86-
jobOverrides: rt.array(setupMlModuleLogEntryRateJobOverridesRT),
87-
datafeedOverrides: rt.array(rt.object),
65+
jobOverrides: rt.array(setupMlModuleJobOverridesRT),
66+
datafeedOverrides: rt.array(setupMlModuleDatafeedOverridesRT),
8867
});
8968

9069
const setupMlModuleRequestPayloadRT = rt.intersection([

0 commit comments

Comments
 (0)