Skip to content

Commit 7baab61

Browse files
Wylie Conlonkibanamachine
Wylie Conlon
and
kibanamachine
committed
[Lens] Create mathColumn function to improve performance (#101908)
* [Lens] Create mathColumn function to improve performance * Fix empty formula case * Fix tinymath memoization Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
1 parent 0db70be commit 7baab61

File tree

8 files changed

+248
-46
lines changed

8 files changed

+248
-46
lines changed

‎docs/canvas/canvas-function-reference.asciidoc‎

Lines changed: 49 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ Alias: `condition`
7171
[[alterColumn_fn]]
7272
=== `alterColumn`
7373

74-
Converts between core types, including `string`, `number`, `null`, `boolean`, and `date`, and renames columns. See also <<mapColumn_fn>> and <<staticColumn_fn>>.
74+
Converts between core types, including `string`, `number`, `null`, `boolean`, and `date`, and renames columns. See also <<mapColumn_fn>>, <<mathColumn_fn>>, and <<staticColumn_fn>>.
7575

7676
*Expression syntax*
7777
[source,js]
@@ -1717,23 +1717,23 @@ Adds a column calculated as the result of other columns. Changes are made only w
17171717
|===
17181718
|Argument |Type |Description
17191719

1720+
|`id`
1721+
1722+
|`string`, `null`
1723+
|An optional id of the resulting column. When no id is provided, the id will be looked up from the existing column by the provided name argument. If no column with this name exists yet, a new column with this name and an identical id will be added to the table.
1724+
17201725
|_Unnamed_ ***
17211726

17221727
Aliases: `column`, `name`
17231728
|`string`
1724-
|The name of the resulting column.
1729+
|The name of the resulting column. Names are not required to be unique.
17251730

17261731
|`expression` ***
17271732

17281733
Aliases: `exp`, `fn`, `function`
17291734
|`boolean`, `number`, `string`, `null`
17301735
|A Canvas expression that is passed to each row as a single row `datatable`.
17311736

1732-
|`id`
1733-
1734-
|`string`, `null`
1735-
|An optional id of the resulting column. When not specified or `null` the name argument is used as id.
1736-
17371737
|`copyMetaFrom`
17381738

17391739
|`string`, `null`
@@ -1808,6 +1808,47 @@ Default: `"throw"`
18081808
*Returns:* `number` | `boolean` | `null`
18091809

18101810

1811+
[float]
1812+
[[mathColumn_fn]]
1813+
=== `mathColumn`
1814+
1815+
Adds a column by evaluating `TinyMath` on each row. This function is optimized for math, so it performs better than the <<mapColumn_fn>> with a <<math_fn>>.
1816+
*Accepts:* `datatable`
1817+
1818+
[cols="3*^<"]
1819+
|===
1820+
|Argument |Type |Description
1821+
1822+
|id ***
1823+
|`string`
1824+
|id of the resulting column. Must be unique.
1825+
1826+
|name ***
1827+
|`string`
1828+
|The name of the resulting column. Names are not required to be unique.
1829+
1830+
|_Unnamed_
1831+
1832+
Alias: `expression`
1833+
|`string`
1834+
|A `TinyMath` expression evaluated on each row. See https://www.elastic.co/guide/en/kibana/current/canvas-tinymath-functions.html.
1835+
1836+
|`onError`
1837+
1838+
|`string`
1839+
|In case the `TinyMath` evaluation fails or returns NaN, the return value is specified by onError. For example, `"null"`, `"zero"`, `"false"`, `"throw"`. When `"throw"`, it will throw an exception, terminating expression execution.
1840+
1841+
Default: `"throw"`
1842+
1843+
|`copyMetaFrom`
1844+
1845+
|`string`, `null`
1846+
|If set, the meta object from the specified column id is copied over to the specified target column. Throws an exception if the column doesn't exist
1847+
|===
1848+
1849+
*Returns:* `datatable`
1850+
1851+
18111852
[float]
18121853
[[metric_fn]]
18131854
=== `metric`
@@ -2581,7 +2622,7 @@ Default: `false`
25812622
[[staticColumn_fn]]
25822623
=== `staticColumn`
25832624

2584-
Adds a column with the same static value in every row. See also <<alterColumn_fn>> and <<mapColumn_fn>>.
2625+
Adds a column with the same static value in every row. See also <<alterColumn_fn>>, <<mapColumn_fn>>, and <<mathColumn_fn>>.
25852626

25862627
*Accepts:* `datatable`
25872628

‎packages/kbn-tinymath/src/index.js‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,11 @@
77
*/
88

99
const { get } = require('lodash');
10+
const memoizeOne = require('memoize-one');
1011
// eslint-disable-next-line import/no-unresolved
1112
const { parse: parseFn } = require('../grammar');
1213
const { functions: includedFunctions } = require('./functions');
1314

14-
module.exports = { parse, evaluate, interpret };
15-
1615
function parse(input, options) {
1716
if (input == null) {
1817
throw new Error('Missing expression');
@@ -29,9 +28,11 @@ function parse(input, options) {
2928
}
3029
}
3130

31+
const memoizedParse = memoizeOne(parse);
32+
3233
function evaluate(expression, scope = {}, injectedFunctions = {}) {
3334
scope = scope || {};
34-
return interpret(parse(expression), scope, injectedFunctions);
35+
return interpret(memoizedParse(expression), scope, injectedFunctions);
3536
}
3637

3738
function interpret(node, scope, injectedFunctions) {
@@ -79,3 +80,5 @@ function isOperable(args) {
7980
return typeof arg === 'number' && !isNaN(arg);
8081
});
8182
}
83+
84+
module.exports = { parse: memoizedParse, evaluate, interpret };

‎src/plugins/expressions/common/expression_functions/specs/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@ export * from './moving_average';
1818
export * from './ui_setting';
1919
export { mapColumn, MapColumnArguments } from './map_column';
2020
export { math, MathArguments, MathInput } from './math';
21+
export { mathColumn, MathColumnArguments } from './math_column';

‎src/plugins/expressions/common/expression_functions/specs/math_column.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import { i18n } from '@kbn/i18n';
10+
import { ExpressionFunctionDefinition } from '../types';
11+
import { math, MathArguments } from './math';
12+
import { Datatable, DatatableColumn, getType } from '../../expression_types';
13+
14+
export type MathColumnArguments = MathArguments & {
15+
id: string;
16+
name?: string;
17+
copyMetaFrom?: string | null;
18+
};
19+
20+
export const mathColumn: ExpressionFunctionDefinition<
21+
'mathColumn',
22+
Datatable,
23+
MathColumnArguments,
24+
Datatable
25+
> = {
26+
name: 'mathColumn',
27+
type: 'datatable',
28+
inputTypes: ['datatable'],
29+
help: i18n.translate('expressions.functions.mathColumnHelpText', {
30+
defaultMessage:
31+
'Adds a column calculated as the result of other columns. ' +
32+
'Changes are made only when you provide arguments.' +
33+
'See also {alterColumnFn} and {staticColumnFn}.',
34+
values: {
35+
alterColumnFn: '`alterColumn`',
36+
staticColumnFn: '`staticColumn`',
37+
},
38+
}),
39+
args: {
40+
...math.args,
41+
id: {
42+
types: ['string'],
43+
help: i18n.translate('expressions.functions.mathColumn.args.idHelpText', {
44+
defaultMessage: 'id of the resulting column. Must be unique.',
45+
}),
46+
required: true,
47+
},
48+
name: {
49+
types: ['string'],
50+
aliases: ['_', 'column'],
51+
help: i18n.translate('expressions.functions.mathColumn.args.nameHelpText', {
52+
defaultMessage: 'The name of the resulting column. Names are not required to be unique.',
53+
}),
54+
required: true,
55+
},
56+
copyMetaFrom: {
57+
types: ['string', 'null'],
58+
help: i18n.translate('expressions.functions.mathColumn.args.copyMetaFromHelpText', {
59+
defaultMessage:
60+
"If set, the meta object from the specified column id is copied over to the specified target column. If the column doesn't exist it silently fails.",
61+
}),
62+
required: false,
63+
default: null,
64+
},
65+
},
66+
fn: (input, args, context) => {
67+
const columns = [...input.columns];
68+
const existingColumnIndex = columns.findIndex(({ id }) => {
69+
return id === args.id;
70+
});
71+
if (existingColumnIndex > -1) {
72+
throw new Error('ID must be unique');
73+
}
74+
75+
const newRows = input.rows.map((row) => {
76+
return {
77+
...row,
78+
[args.id]: math.fn(
79+
{
80+
type: 'datatable',
81+
columns: input.columns,
82+
rows: [row],
83+
},
84+
{
85+
expression: args.expression,
86+
onError: args.onError,
87+
},
88+
context
89+
),
90+
};
91+
});
92+
const type = newRows.length ? getType(newRows[0][args.id]) : 'null';
93+
const newColumn: DatatableColumn = {
94+
id: args.id,
95+
name: args.name ?? args.id,
96+
meta: { type, params: { id: type } },
97+
};
98+
if (args.copyMetaFrom) {
99+
const metaSourceFrom = columns.find(({ id }) => id === args.copyMetaFrom);
100+
newColumn.meta = { ...newColumn.meta, ...(metaSourceFrom?.meta || {}) };
101+
}
102+
103+
columns.push(newColumn);
104+
105+
return {
106+
type: 'datatable',
107+
columns,
108+
rows: newRows,
109+
} as Datatable;
110+
},
111+
};

‎src/plugins/expressions/common/expression_functions/specs/tests/math_column.test.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
import { mathColumn } from '../math_column';
10+
import { functionWrapper, testTable } from './utils';
11+
12+
describe('mathColumn', () => {
13+
const fn = functionWrapper(mathColumn);
14+
15+
it('throws if the id is used', () => {
16+
expect(() => fn(testTable, { id: 'price', name: 'price', expression: 'price * 2' })).toThrow(
17+
`ID must be unique`
18+
);
19+
});
20+
21+
it('applies math to each row by id', () => {
22+
const result = fn(testTable, { id: 'output', name: 'output', expression: 'quantity * price' });
23+
expect(result.columns).toEqual([
24+
...testTable.columns,
25+
{ id: 'output', name: 'output', meta: { params: { id: 'number' }, type: 'number' } },
26+
]);
27+
expect(result.rows[0]).toEqual({
28+
in_stock: true,
29+
name: 'product1',
30+
output: 60500,
31+
price: 605,
32+
quantity: 100,
33+
time: 1517842800950,
34+
});
35+
});
36+
37+
it('handles onError', () => {
38+
const args = {
39+
id: 'output',
40+
name: 'output',
41+
expression: 'quantity / 0',
42+
};
43+
expect(() => fn(testTable, args)).toThrowError(`Cannot divide by 0`);
44+
expect(() => fn(testTable, { ...args, onError: 'throw' })).toThrow();
45+
expect(fn(testTable, { ...args, onError: 'zero' }).rows[0].output).toEqual(0);
46+
expect(fn(testTable, { ...args, onError: 'false' }).rows[0].output).toEqual(false);
47+
expect(fn(testTable, { ...args, onError: 'null' }).rows[0].output).toEqual(null);
48+
});
49+
50+
it('should copy over the meta information from the specified column', async () => {
51+
const result = await fn(
52+
{
53+
...testTable,
54+
columns: [
55+
...testTable.columns,
56+
{
57+
id: 'myId',
58+
name: 'myName',
59+
meta: { type: 'date', params: { id: 'number', params: { digits: 2 } } },
60+
},
61+
],
62+
rows: testTable.rows.map((row) => ({ ...row, myId: Date.now() })),
63+
},
64+
{ id: 'output', name: 'name', copyMetaFrom: 'myId', expression: 'price + 2' }
65+
);
66+
67+
expect(result.type).toBe('datatable');
68+
expect(result.columns[result.columns.length - 1]).toEqual({
69+
id: 'output',
70+
name: 'name',
71+
meta: { type: 'date', params: { id: 'number', params: { digits: 2 } } },
72+
});
73+
});
74+
});

‎src/plugins/expressions/common/service/expressions_services.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import {
3131
mapColumn,
3232
overallMetric,
3333
math,
34+
mathColumn,
3435
} from '../expression_functions';
3536

3637
/**
@@ -344,6 +345,7 @@ export class ExpressionsService implements PersistableStateService<ExpressionAst
344345
overallMetric,
345346
mapColumn,
346347
math,
348+
mathColumn,
347349
]) {
348350
this.registerFunction(fn);
349351
}

‎x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.tsx‎

Lines changed: 2 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -100,28 +100,11 @@ export const formulaOperation: OperationDefinition<
100100
return [
101101
{
102102
type: 'function',
103-
function: 'mapColumn',
103+
function: currentColumn.references.length ? 'mathColumn' : 'mapColumn',
104104
arguments: {
105105
id: [columnId],
106106
name: [label || defaultLabel],
107-
exp: [
108-
{
109-
type: 'expression',
110-
chain: currentColumn.references.length
111-
? [
112-
{
113-
type: 'function',
114-
function: 'math',
115-
arguments: {
116-
expression: [
117-
currentColumn.references.length ? `"${currentColumn.references[0]}"` : ``,
118-
],
119-
},
120-
},
121-
]
122-
: [],
123-
},
124-
],
107+
expression: [currentColumn.references.length ? `"${currentColumn.references[0]}"` : ''],
125108
},
126109
},
127110
];

‎x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/math.tsx‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -45,25 +45,12 @@ export const mathOperation: OperationDefinition<MathIndexPatternColumn, 'managed
4545
return [
4646
{
4747
type: 'function',
48-
function: 'mapColumn',
48+
function: 'mathColumn',
4949
arguments: {
5050
id: [columnId],
5151
name: [columnId],
52-
exp: [
53-
{
54-
type: 'expression',
55-
chain: [
56-
{
57-
type: 'function',
58-
function: 'math',
59-
arguments: {
60-
expression: [astToString(column.params.tinymathAst)],
61-
onError: ['null'],
62-
},
63-
},
64-
],
65-
},
66-
],
52+
expression: [astToString(column.params.tinymathAst)],
53+
onError: ['null'],
6754
},
6855
},
6956
];

0 commit comments

Comments
 (0)