Skip to content

Commit 63ac0f7

Browse files
qn895qn895
authored
[ML] Add better UI support for runtime fields Transforms (#90363)
* [ML] Add RT support for transforms from index pattern * [ML] Add support for cloned transform from api * [ML] Add support for runtime pivot * [ML] Add support for api created runtime * [ML] Add preview for expanded row * [ML] Add runtime fields to dropdown options * [ML] Add runtime fields to latest * [ML] Fix duplicate columns * [ML] Update types and test * [ML] Add runtime mappings to index pattern on creation * [ML] Add callout to show unsupported fields in dfa * [ML] Update types to RuntimeField * [ML] Fix runtime fields, remove runtime mappings, fix copy to console * [ML] Fix incompatible kbn field type * [ML] Add advanced mappings editor * [ML] Add support for filter terms agg control * [ML] Fix jest tests hanging * [ML] Fix translations * [ML] Fix over-sized buttons for filter range * [ML] Update runtime mappings schema * [ML] Update runtime mappings schema * [ML] Use isRecord for object checks * [ML] Fix and more message * [ML] Update schema to correctly match types * [ML] Update schema to correctly match types * [ML] Fix pivot duplicates * [ML] Rename isRecord to isPopulatedObject * [ML] Remove fit-content * [ML] Update runtime field type to prevent potential conflicts * Revert "[ML] Remove fit-content" This reverts commit 76c9c79 * [ML] Remove misc comment * [ML] Fix missing typeof * [ML] Add sorts and constants * [ML] Add i18n to includedFields description * [ML] fix imports * [ML] Only pass runtime mappings if it's latest * [ML] Fix functional tests
1 parent 540b1d3 commit 63ac0f7

48 files changed

Lines changed: 1224 additions & 145 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎x-pack/plugins/ml/common/types/feature_importance.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
* 2.0.
66
*/
77

8+
import { isPopulatedObject } from '../util/object_utils';
9+
810
export type FeatureImportanceClassName = string | number | boolean;
911

1012
export interface ClassFeatureImportance {
@@ -87,7 +89,7 @@ export function isClassificationFeatureImportanceBaseline(
8789
baselineData: any
8890
): baselineData is ClassificationFeatureImportanceBaseline {
8991
return (
90-
typeof baselineData === 'object' &&
92+
isPopulatedObject(baselineData) &&
9193
baselineData.hasOwnProperty('classes') &&
9294
Array.isArray(baselineData.classes)
9395
);
@@ -96,5 +98,5 @@ export function isClassificationFeatureImportanceBaseline(
9698
export function isRegressionFeatureImportanceBaseline(
9799
baselineData: any
98100
): baselineData is RegressionFeatureImportanceBaseline {
99-
return typeof baselineData === 'object' && baselineData.hasOwnProperty('baseline');
101+
return isPopulatedObject(baselineData) && baselineData.hasOwnProperty('baseline');
100102
}

‎x-pack/plugins/ml/common/types/fields.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* 2.0.
66
*/
77

8-
import { ES_FIELD_TYPES, RuntimeField } from '../../../../../src/plugins/data/common';
8+
import { ES_FIELD_TYPES } from '../../../../../src/plugins/data/common';
99
import {
1010
ML_JOB_AGGREGATION,
1111
KIBANA_AGGREGATION,
@@ -106,4 +106,18 @@ export interface AggCardinality {
106106
}
107107

108108
export type RollupFields = Record<FieldId, [Record<'agg', ES_AGGREGATION>]>;
109+
110+
// Replace this with import once #88995 is merged
111+
const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
112+
type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
113+
114+
export interface RuntimeField {
115+
type: RuntimeType;
116+
script:
117+
| string
118+
| {
119+
source: string;
120+
};
121+
}
122+
109123
export type RuntimeMappings = Record<string, RuntimeField>;

‎x-pack/plugins/ml/common/util/datafeed_utils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export const getDatafeedAggregations = (
2020
};
2121

2222
export const getAggregationBucketsName = (aggregations: any): string | undefined => {
23-
if (typeof aggregations === 'object') {
23+
if (aggregations !== null && typeof aggregations === 'object') {
2424
const keys = Object.keys(aggregations);
2525
return keys.length > 0 ? keys[0] : undefined;
2626
}

‎x-pack/plugins/ml/common/util/job_utils.ts‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import {
2828
getDatafeedAggregations,
2929
} from './datafeed_utils';
3030
import { findAggField } from './validation_utils';
31+
import { isPopulatedObject } from './object_utils';
3132

3233
export interface ValidationResults {
3334
valid: boolean;
@@ -51,17 +52,9 @@ export function calculateDatafeedFrequencyDefaultSeconds(bucketSpanSeconds: numb
5152
}
5253

5354
export function hasRuntimeMappings(job: CombinedJob): boolean {
54-
const hasDatafeed =
55-
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
55+
const hasDatafeed = isPopulatedObject(job.datafeed_config);
5656
if (hasDatafeed) {
57-
const runtimeMappings =
58-
typeof job.datafeed_config.runtime_mappings === 'object'
59-
? Object.keys(job.datafeed_config.runtime_mappings)
60-
: undefined;
61-
62-
if (Array.isArray(runtimeMappings) && runtimeMappings.length > 0) {
63-
return true;
64-
}
57+
return isPopulatedObject(job.datafeed_config.runtime_mappings);
6558
}
6659
return false;
6760
}
@@ -114,7 +107,11 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
114107
// If the datafeed uses script fields, we can only plot the time series if
115108
// model plot is enabled. Without model plot it will be very difficult or impossible
116109
// to invert to a reverse search of the underlying metric data.
117-
if (isSourceDataChartable === true && typeof job.datafeed_config?.script_fields === 'object') {
110+
if (
111+
isSourceDataChartable === true &&
112+
job.datafeed_config?.script_fields !== null &&
113+
typeof job.datafeed_config?.script_fields === 'object'
114+
) {
118115
// Perform extra check to see if the detector is using a scripted field.
119116
const scriptFields = Object.keys(job.datafeed_config.script_fields);
120117
isSourceDataChartable =
@@ -123,8 +120,7 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
123120
scriptFields.indexOf(dtr.over_field_name!) === -1;
124121
}
125122

126-
const hasDatafeed =
127-
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
123+
const hasDatafeed = isPopulatedObject(job.datafeed_config);
128124
if (hasDatafeed) {
129125
// We cannot plot the source data for some specific aggregation configurations
130126
const aggs = getDatafeedAggregations(job.datafeed_config);

‎x-pack/plugins/ml/common/util/object_utils.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
export const isPopulatedObject = <T = Record<string, any>>(arg: any): arg is T => {
9+
return typeof arg === 'object' && arg !== null && Object.keys(arg).length > 0;
10+
};

‎x-pack/plugins/ml/common/util/validation_utils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ export function findAggField(
4545
value = returnParent === true ? aggs : aggs[k];
4646
return true;
4747
}
48-
if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') {
48+
if (aggs.hasOwnProperty(k) && aggs[k] !== null && typeof aggs[k] === 'object') {
4949
value = findAggField(aggs[k], fieldName, returnParent);
5050
return value !== undefined;
5151
}

‎x-pack/plugins/ml/public/application/components/data_grid/common.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ import { getNestedProperty } from '../../util/object_utils';
4848
import { mlFieldFormatService } from '../../services/field_format_service';
4949

5050
import { DataGridItem, IndexPagination, RenderCellValue } from './types';
51+
import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns';
52+
import { RuntimeMappings } from '../../../../common/types/fields';
53+
import { isPopulatedObject } from '../../../../common/util/object_utils';
5154

5255
export const INIT_MAX_COLUMNS = 10;
5356

@@ -86,6 +89,37 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str
8689
return indexPatternFields;
8790
};
8891

92+
/**
93+
* Return a map of runtime_mappings for each of the index pattern field provided
94+
* to provide in ES search queries
95+
* @param indexPatternFields
96+
* @param indexPattern
97+
* @param clonedRuntimeMappings
98+
*/
99+
export const getRuntimeFieldsMapping = (
100+
indexPatternFields: string[] | undefined,
101+
indexPattern: IndexPattern | undefined,
102+
clonedRuntimeMappings?: RuntimeMappings
103+
) => {
104+
if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {};
105+
const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
106+
let combinedRuntimeMappings: RuntimeMappings = {};
107+
108+
if (isPopulatedObject(ipRuntimeMappings)) {
109+
indexPatternFields.forEach((ipField) => {
110+
if (ipRuntimeMappings.hasOwnProperty(ipField)) {
111+
combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField];
112+
}
113+
});
114+
}
115+
if (isPopulatedObject(clonedRuntimeMappings)) {
116+
combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings };
117+
}
118+
return Object.keys(combinedRuntimeMappings).length > 0
119+
? { runtime_mappings: combinedRuntimeMappings }
120+
: {};
121+
};
122+
89123
export interface FieldTypes {
90124
[key: string]: ES_FIELD_TYPES;
91125
}
@@ -135,6 +169,45 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
135169
};
136170

137171
export const NON_AGGREGATABLE = 'non-aggregatable';
172+
173+
export const getDataGridSchemaFromESFieldType = (
174+
fieldType: ES_FIELD_TYPES | undefined | RuntimeField['type']
175+
): string | undefined => {
176+
// Built-in values are ['boolean', 'currency', 'datetime', 'numeric', 'json']
177+
// To fall back to the default string schema it needs to be undefined.
178+
let schema;
179+
180+
switch (fieldType) {
181+
case ES_FIELD_TYPES.GEO_POINT:
182+
case ES_FIELD_TYPES.GEO_SHAPE:
183+
schema = 'json';
184+
break;
185+
case ES_FIELD_TYPES.BOOLEAN:
186+
schema = 'boolean';
187+
break;
188+
case ES_FIELD_TYPES.DATE:
189+
case ES_FIELD_TYPES.DATE_NANOS:
190+
schema = 'datetime';
191+
break;
192+
case ES_FIELD_TYPES.BYTE:
193+
case ES_FIELD_TYPES.DOUBLE:
194+
case ES_FIELD_TYPES.FLOAT:
195+
case ES_FIELD_TYPES.HALF_FLOAT:
196+
case ES_FIELD_TYPES.INTEGER:
197+
case ES_FIELD_TYPES.LONG:
198+
case ES_FIELD_TYPES.SCALED_FLOAT:
199+
case ES_FIELD_TYPES.SHORT:
200+
schema = 'numeric';
201+
break;
202+
// keep schema undefined for text based columns
203+
case ES_FIELD_TYPES.KEYWORD:
204+
case ES_FIELD_TYPES.TEXT:
205+
break;
206+
}
207+
208+
return schema;
209+
};
210+
138211
export const getDataGridSchemaFromKibanaFieldType = (
139212
field: IFieldType | undefined
140213
): string | undefined => {

‎x-pack/plugins/ml/public/application/components/data_grid/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77

88
export {
99
getDataGridSchemasFromFieldTypes,
10+
getDataGridSchemaFromESFieldType,
1011
getDataGridSchemaFromKibanaFieldType,
1112
getFieldsFromKibanaIndexPattern,
13+
getRuntimeFieldsMapping,
1214
multiColumnSortFactory,
1315
showDataGridColumnChartErrorMessageToast,
1416
useRenderCellValue,

‎x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_details.tsx‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,16 @@ export const ConfigurationStepDetails: FC<Props> = ({ setCurrentStep, state }) =
6969
}),
7070
description:
7171
includes.length > MAX_INCLUDES_LENGTH
72-
? `${includes.slice(0, MAX_INCLUDES_LENGTH).join(', ')} ... (and ${
73-
includes.length - MAX_INCLUDES_LENGTH
74-
} more)`
72+
? i18n.translate(
73+
'xpack.ml.dataframe.analytics.create.configDetails.includedFieldsAndMoreDescription',
74+
{
75+
defaultMessage: '{includedFields} ... (and {extraCount} more)',
76+
values: {
77+
extraCount: includes.length - MAX_INCLUDES_LENGTH,
78+
includedFields: includes.slice(0, MAX_INCLUDES_LENGTH).join(', '),
79+
},
80+
}
81+
)
7582
: includes.join(', '),
7683
},
7784
];

‎x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react';
99
import {
1010
EuiBadge,
11+
EuiCallOut,
1112
EuiComboBox,
1213
EuiComboBoxOptionOption,
1314
EuiFormRow,
@@ -19,6 +20,7 @@ import {
1920
import { i18n } from '@kbn/i18n';
2021
import { debounce } from 'lodash';
2122

23+
import { FormattedMessage } from '@kbn/i18n/react';
2224
import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
2325
import { useMlContext } from '../../../../../contexts/ml';
2426

@@ -62,6 +64,8 @@ const requiredFieldsErrorText = i18n.translate(
6264
}
6365
);
6466

67+
const maxRuntimeFieldsDisplayCount = 5;
68+
6569
export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
6670
actions,
6771
state,
@@ -314,6 +318,15 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
314318
};
315319
}, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]);
316320

321+
const unsupportedRuntimeFields = useMemo(
322+
() =>
323+
currentIndexPattern.fields
324+
.getAll()
325+
.filter((f) => f.runtimeField)
326+
.map((f) => `'${f.displayName}'`),
327+
[currentIndexPattern.fields]
328+
);
329+
317330
return (
318331
<Fragment>
319332
<Messages messages={requestMessages} />
@@ -445,6 +458,36 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
445458
>
446459
<Fragment />
447460
</EuiFormRow>
461+
{Array.isArray(unsupportedRuntimeFields) && unsupportedRuntimeFields.length > 0 && (
462+
<>
463+
<EuiCallOut size="s" color="warning">
464+
<FormattedMessage
465+
id="xpack.ml.dataframe.analytics.create.unsupportedRuntimeFieldsCallout"
466+
defaultMessage="The runtime {runtimeFieldsCount, plural, one {field} other {fields}} {unsupportedRuntimeFields} {extraCountMsg} are not supported for analysis."
467+
values={{
468+
runtimeFieldsCount: unsupportedRuntimeFields.length,
469+
extraCountMsg:
470+
unsupportedRuntimeFields.length - maxRuntimeFieldsDisplayCount > 0 ? (
471+
<FormattedMessage
472+
id="xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg"
473+
defaultMessage="and {count} more"
474+
values={{
475+
count: unsupportedRuntimeFields.length - maxRuntimeFieldsDisplayCount,
476+
}}
477+
/>
478+
) : (
479+
''
480+
),
481+
unsupportedRuntimeFields: unsupportedRuntimeFields
482+
.slice(0, maxRuntimeFieldsDisplayCount)
483+
.join(', '),
484+
}}
485+
/>
486+
</EuiCallOut>
487+
<EuiSpacer />
488+
</>
489+
)}
490+
448491
<AnalysisFieldsTable
449492
dependentVariable={dependentVariable}
450493
includes={includes}

0 commit comments

Comments
 (0)