Skip to content

Commit 7fd7c82

Browse files
darnautovdarnautov
committed
[ML] Inference models management (#74978)
* [ML] init tabs * [ML] init inference API service in UI * [ML] server-side routes * [ML] basic table * [ML] support deletion * [ML] delete multiple models * [ML] WIP expanded row * [ML] fix types * [ML] expanded row * [ML] fix types * [ML] fix i18n id * [ML] change server-side permission check * [ML] refactor types * [ML] show success toast on model deletion, fix models counter * [ML] update expanded row * [ML] pipelines stats * [ML] use refresh observable * [ML] endpoint to fetch associated pipelines * [ML] update the endpoint to fetch associated pipelines * [ML] show pipelines definition in expanded row * [ML] change stats layout * [ML] fix headers * [ML] change breadcrumb title * [ML] fetch models config with pipelines * [ML] change default size to 1000 * [ML] fix collections keys, fix double fetch on initial page load * [ML] adjust models deletion text * [ML] fix DFA jobs on the management page * [ML] small tabs in expanded row * [ML] fix headers text * [ML] fix models fetching without pipelines get permissions * [ML] stats rendering as a description list * [ML] fix i18n id * [ML] remove an extra copyright comment, add selectable messages * [ML] update stats on refresh
1 parent 8946b68 commit 7fd7c82

28 files changed

Lines changed: 1678 additions & 96 deletions

File tree

‎x-pack/plugins/ml/common/types/data_frame_analytics.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,77 @@
55
*/
66

77
import { CustomHttpResponseOptions, ResponseError } from 'kibana/server';
8+
89
export interface DeleteDataFrameAnalyticsWithIndexStatus {
910
success: boolean;
1011
error?: CustomHttpResponseOptions<ResponseError>;
1112
}
13+
14+
export type IndexName = string;
15+
export type DataFrameAnalyticsId = string;
16+
17+
export interface OutlierAnalysis {
18+
[key: string]: {};
19+
20+
outlier_detection: {};
21+
}
22+
23+
interface Regression {
24+
dependent_variable: string;
25+
training_percent?: number;
26+
num_top_feature_importance_values?: number;
27+
prediction_field_name?: string;
28+
}
29+
30+
interface Classification {
31+
dependent_variable: string;
32+
training_percent?: number;
33+
num_top_classes?: string;
34+
num_top_feature_importance_values?: number;
35+
prediction_field_name?: string;
36+
}
37+
38+
export interface RegressionAnalysis {
39+
[key: string]: Regression;
40+
41+
regression: Regression;
42+
}
43+
44+
export interface ClassificationAnalysis {
45+
[key: string]: Classification;
46+
47+
classification: Classification;
48+
}
49+
50+
interface GenericAnalysis {
51+
[key: string]: Record<string, any>;
52+
}
53+
54+
export type AnalysisConfig =
55+
| OutlierAnalysis
56+
| RegressionAnalysis
57+
| ClassificationAnalysis
58+
| GenericAnalysis;
59+
60+
export interface DataFrameAnalyticsConfig {
61+
id: DataFrameAnalyticsId;
62+
description?: string;
63+
dest: {
64+
index: IndexName;
65+
results_field: string;
66+
};
67+
source: {
68+
index: IndexName | IndexName[];
69+
query?: any;
70+
};
71+
analysis: AnalysisConfig;
72+
analyzed_fields: {
73+
includes: string[];
74+
excludes: string[];
75+
};
76+
model_memory_limit: string;
77+
max_num_threads?: number;
78+
create_time: number;
79+
version: string;
80+
allow_lazy_start?: boolean;
81+
}

‎x-pack/plugins/ml/common/types/inference.ts‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
import { DataFrameAnalyticsConfig } from './data_frame_analytics';
8+
9+
export interface IngestStats {
10+
count: number;
11+
time_in_millis: number;
12+
current: number;
13+
failed: number;
14+
}
15+
16+
export interface TrainedModelStat {
17+
model_id: string;
18+
pipeline_count: number;
19+
inference_stats?: {
20+
failure_count: number;
21+
inference_count: number;
22+
cache_miss_count: number;
23+
missing_all_fields_count: number;
24+
timestamp: number;
25+
};
26+
ingest?: {
27+
total: IngestStats;
28+
pipelines: Record<
29+
string,
30+
IngestStats & {
31+
processors: Array<
32+
Record<
33+
string,
34+
{
35+
// TODO use type from ingest_pipelines plugin
36+
type: string;
37+
stats: IngestStats;
38+
}
39+
>
40+
>;
41+
}
42+
>;
43+
};
44+
}
45+
46+
export interface ModelConfigResponse {
47+
created_by: string;
48+
create_time: string;
49+
default_field_map: Record<string, string>;
50+
estimated_heap_memory_usage_bytes: number;
51+
estimated_operations: number;
52+
license_level: string;
53+
metadata?:
54+
| {
55+
analytics_config: DataFrameAnalyticsConfig;
56+
input: any;
57+
}
58+
| Record<string, any>;
59+
model_id: string;
60+
tags: string;
61+
version: string;
62+
inference_config?: Record<string, any>;
63+
pipelines?: Record<string, PipelineDefinition> | null;
64+
}
65+
66+
export interface PipelineDefinition {
67+
processors?: Array<Record<string, any>>;
68+
description?: string;
69+
}
70+
71+
export interface ModelPipelines {
72+
model_id: string;
73+
pipelines: Record<string, PipelineDefinition>;
74+
}
75+
76+
/**
77+
* Get inference response from the ES endpoint
78+
*/
79+
export interface InferenceConfigResponse {
80+
trained_model_configs: ModelConfigResponse[];
81+
}

‎x-pack/plugins/ml/public/application/components/stats_bar/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66

7-
export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats } from './stats_bar';
7+
export { StatsBar, AnalyticStatsBarStats, JobStatsBarStats, ModelsBarStats } from './stats_bar';

‎x-pack/plugins/ml/public/application/components/stats_bar/stats_bar.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,11 @@ export interface AnalyticStatsBarStats extends Stats {
2323
stopped: StatsBarStat;
2424
}
2525

26-
export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats;
26+
export interface ModelsBarStats {
27+
total: StatsBarStat;
28+
}
29+
30+
export type StatsBarStats = JobStatsBarStats | AnalyticStatsBarStats | ModelsBarStats;
2731
type StatsKey = keyof StatsBarStats;
2832

2933
interface StatsBarProps {

‎x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts‎

Lines changed: 18 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,21 @@
55
*/
66

77
import { useEffect } from 'react';
8-
import { BehaviorSubject } from 'rxjs';
9-
import { filter, distinctUntilChanged } from 'rxjs/operators';
10-
import { Subscription } from 'rxjs';
8+
import { BehaviorSubject, Subscription } from 'rxjs';
9+
import { distinctUntilChanged, filter } from 'rxjs/operators';
1110
import { cloneDeep } from 'lodash';
1211
import { ml } from '../../services/ml_api_service';
1312
import { Dictionary } from '../../../../common/types/common';
1413
import { getErrorMessage } from '../../../../common/util/errors';
1514
import { SavedSearchQuery } from '../../contexts/ml';
15+
import {
16+
AnalysisConfig,
17+
ClassificationAnalysis,
18+
OutlierAnalysis,
19+
RegressionAnalysis,
20+
} from '../../../../common/types/data_frame_analytics';
1621

17-
export type IndexName = string;
1822
export type IndexPattern = string;
19-
export type DataFrameAnalyticsId = string;
2023

2124
export enum ANALYSIS_CONFIG_TYPE {
2225
OUTLIER_DETECTION = 'outlier_detection',
@@ -46,34 +49,6 @@ export enum OUTLIER_ANALYSIS_METHOD {
4649
DISTANCE_KNN = 'distance_knn',
4750
}
4851

49-
interface OutlierAnalysis {
50-
[key: string]: {};
51-
outlier_detection: {};
52-
}
53-
54-
interface Regression {
55-
dependent_variable: string;
56-
training_percent?: number;
57-
num_top_feature_importance_values?: number;
58-
prediction_field_name?: string;
59-
}
60-
export interface RegressionAnalysis {
61-
[key: string]: Regression;
62-
regression: Regression;
63-
}
64-
65-
interface Classification {
66-
dependent_variable: string;
67-
training_percent?: number;
68-
num_top_classes?: string;
69-
num_top_feature_importance_values?: number;
70-
prediction_field_name?: string;
71-
}
72-
export interface ClassificationAnalysis {
73-
[key: string]: Classification;
74-
classification: Classification;
75-
}
76-
7752
export interface LoadExploreDataArg {
7853
filterByIsTraining?: boolean;
7954
searchQuery: SavedSearchQuery;
@@ -165,22 +140,12 @@ export interface ClassificationEvaluateResponse {
165140
};
166141
}
167142

168-
interface GenericAnalysis {
169-
[key: string]: Record<string, any>;
170-
}
171-
172143
interface LoadEvaluateResult {
173144
success: boolean;
174145
eval: RegressionEvaluateResponse | ClassificationEvaluateResponse | null;
175146
error: string | null;
176147
}
177148

178-
type AnalysisConfig =
179-
| OutlierAnalysis
180-
| RegressionAnalysis
181-
| ClassificationAnalysis
182-
| GenericAnalysis;
183-
184149
export const getAnalysisType = (analysis: AnalysisConfig): string => {
185150
const keys = Object.keys(analysis);
186151

@@ -342,29 +307,6 @@ export interface UpdateDataFrameAnalyticsConfig {
342307
max_num_threads?: number;
343308
}
344309

345-
export interface DataFrameAnalyticsConfig {
346-
id: DataFrameAnalyticsId;
347-
description?: string;
348-
dest: {
349-
index: IndexName;
350-
results_field: string;
351-
};
352-
source: {
353-
index: IndexName | IndexName[];
354-
query?: any;
355-
};
356-
analysis: AnalysisConfig;
357-
analyzed_fields: {
358-
includes: string[];
359-
excludes: string[];
360-
};
361-
model_memory_limit: string;
362-
max_num_threads?: number;
363-
create_time: number;
364-
version: string;
365-
allow_lazy_start?: boolean;
366-
}
367-
368310
export enum REFRESH_ANALYTICS_LIST_STATE {
369311
ERROR = 'error',
370312
IDLE = 'idle',
@@ -379,21 +321,26 @@ export const useRefreshAnalyticsList = (
379321
callback: {
380322
isLoading?(d: boolean): void;
381323
onRefresh?(): void;
382-
} = {}
324+
} = {},
325+
isManagementTable = false
383326
) => {
384327
useEffect(() => {
385328
const distinct$ = refreshAnalyticsList$.pipe(distinctUntilChanged());
386329

387330
const subscriptions: Subscription[] = [];
388331

389332
if (typeof callback.onRefresh === 'function') {
390-
// initial call to refresh
391-
callback.onRefresh();
333+
// required in order to fetch the DFA jobs on the management page
334+
if (isManagementTable) callback.onRefresh();
392335

393336
subscriptions.push(
394337
distinct$
395338
.pipe(filter((state) => state === REFRESH_ANALYTICS_LIST_STATE.REFRESH))
396-
.subscribe(() => typeof callback.onRefresh === 'function' && callback.onRefresh())
339+
.subscribe(() => {
340+
if (typeof callback.onRefresh === 'function') {
341+
callback.onRefresh();
342+
}
343+
})
397344
);
398345
}
399346

@@ -410,7 +357,7 @@ export const useRefreshAnalyticsList = (
410357
return () => {
411358
subscriptions.map((sub) => sub.unsubscribe());
412359
};
413-
}, []);
360+
}, [callback.onRefresh]);
414361

415362
return {
416363
refresh: () => {

‎x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,13 @@ import {
1313
isClassificationAnalysis,
1414
isOutlierAnalysis,
1515
isRegressionAnalysis,
16-
DataFrameAnalyticsConfig,
1716
} from './analytics';
1817
import { Field } from '../../../../common/types/fields';
1918
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
2019
import { newJobCapsService } from '../../services/new_job_capabilities_service';
2120

2221
import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants';
22+
import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
2323

2424
export type EsId = string;
2525
export type EsDocSource = Record<string, any>;

‎x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_data.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ import { ml } from '../../services/ml_api_service';
1212
import { isKeywordAndTextType } from '../common/fields';
1313
import { SavedSearchQuery } from '../../contexts/ml';
1414

15-
import { DataFrameAnalyticsConfig, INDEX_STATUS } from './analytics';
15+
import { INDEX_STATUS } from './analytics';
16+
import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
1617

1718
export const getIndexData = async (
1819
jobConfig: DataFrameAnalyticsConfig | undefined,

‎x-pack/plugins/ml/public/application/data_frame_analytics/common/index.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,7 @@ export {
1111
isOutlierAnalysis,
1212
refreshAnalyticsList$,
1313
useRefreshAnalyticsList,
14-
DataFrameAnalyticsId,
15-
DataFrameAnalyticsConfig,
1614
UpdateDataFrameAnalyticsConfig,
17-
IndexName,
1815
IndexPattern,
1916
REFRESH_ANALYTICS_LIST_STATE,
2017
ANALYSIS_CONFIG_TYPE,
@@ -45,3 +42,6 @@ export { getIndexData } from './get_index_data';
4542
export { getIndexFields } from './get_index_fields';
4643

4744
export { useResultsViewConfig } from './use_results_view_config';
45+
export { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
46+
export { DataFrameAnalyticsId } from '../../../../common/types/data_frame_analytics';
47+
export { IndexName } from '../../../../common/types/data_frame_analytics';

‎x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ import { FormattedMessage } from '@kbn/i18n/react';
2323
import { useMlContext } from '../../../contexts/ml';
2424
import { newJobCapsService } from '../../../services/new_job_capabilities_service';
2525
import { ml } from '../../../services/ml_api_service';
26-
import { DataFrameAnalyticsId } from '../../common/analytics';
2726
import { useCreateAnalyticsForm } from '../analytics_management/hooks/use_create_analytics_form';
2827
import { CreateAnalyticsAdvancedEditor } from './components/create_analytics_advanced_editor';
2928
import { AdvancedStep, ConfigurationStep, CreateStep, DetailsStep } from './components';
29+
import { DataFrameAnalyticsId } from '../../../../../common/types/data_frame_analytics';
3030

3131
export enum ANALYTICS_STEPS {
3232
CONFIGURATION,

0 commit comments

Comments
 (0)