Skip to content

Commit 2cd996e

Browse files
zhongnansuzhongnansu
committed
refactor logic for creating dsl from saved object using buildOpensearchQuery()
1 parent 24cea4e commit 2cd996e

4 files changed

Lines changed: 96 additions & 137 deletions

File tree

‎dashboards-reports/.babelrc‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎dashboards-reports/babel.config.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
module.exports = {
10+
presets: [
11+
require('@babel/preset-env', {
12+
targets: { node: '10' },
13+
}),
14+
require('@babel/preset-react'),
15+
require('@babel/preset-typescript'),
16+
],
17+
plugins: [
18+
require('@babel/plugin-proposal-class-properties'),
19+
require('@babel/plugin-proposal-object-rest-spread'),
20+
['@babel/plugin-transform-modules-commonjs', { allowTopLevelThis: true }],
21+
[require('@babel/plugin-transform-runtime'), { regenerator: true }],
22+
],
23+
};

‎dashboards-reports/server/routes/utils/dataReportHelpers.ts‎

Lines changed: 67 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ import converter from 'json-2-csv';
2929
import _ from 'lodash';
3030
import moment from 'moment';
3131
import { DATA_REPORT_CONFIG } from './constants';
32+
import {
33+
buildOpenSearchQuery,
34+
Filter,
35+
Query,
36+
} from '../../../../../src/plugins/data/common';
3237

3338
export var metaData = {
3439
saved_search_id: <string>null,
@@ -42,7 +47,7 @@ export var metaData = {
4247
fields_exist: <boolean>false,
4348
selectedFields: <any>[],
4449
paternName: <string>null,
45-
filters: <any>[],
50+
searchSourceJSON: <any>[],
4651
dateFields: <any>[],
4752
};
4853

@@ -65,93 +70,20 @@ export const getSelectedFields = async (columns) => {
6570

6671
// Build the OpenSearch query from the meta data
6772
// is_count is set to 1 if we building the count query but 0 if we building the fetch data query
68-
export const buildQuery = (report, is_count) => {
69-
let requestBody = esb.boolQuery();
70-
const filters = report._source.filters;
71-
for (let item of JSON.parse(filters).filter) {
72-
if (item.meta.disabled === false) {
73-
switch (item.meta.negate) {
74-
case false:
75-
switch (item.meta.type) {
76-
case 'phrase':
77-
requestBody.must(
78-
esb.matchPhraseQuery(item.meta.key, item.meta.params.query)
79-
);
80-
break;
81-
case 'exists':
82-
requestBody.must(esb.existsQuery(item.meta.key));
83-
break;
84-
case 'phrases':
85-
if (item.meta.value.indexOf(',') > -1) {
86-
const valueSplit = item.meta.value.split(', ');
87-
for (const [key, incr] of valueSplit.entries()) {
88-
requestBody.should(esb.matchPhraseQuery(item.meta.key, incr));
89-
}
90-
} else {
91-
requestBody.should(
92-
esb.matchPhraseQuery(item.meta.key, item.meta.value)
93-
);
94-
}
95-
requestBody.minimumShouldMatch(1);
96-
break;
97-
case 'range':
98-
const builder = esb.rangeQuery(item.meta.key);
99-
if (item.meta.params.gte) builder.gte(item.meta.params.gte);
100-
if (item.meta.params.lte) builder.lte(item.meta.params.lte);
101-
if (item.meta.params.gt) builder.gt(item.meta.params.gt);
102-
if (item.meta.params.lt) builder.lt(item.meta.params.lt);
103-
requestBody.must(builder);
104-
break;
105-
}
106-
break;
107-
case true:
108-
switch (item.meta.type) {
109-
case 'phrase':
110-
requestBody.mustNot(
111-
esb.matchPhraseQuery(item.meta.key, item.meta.params.query)
112-
);
113-
break;
114-
case 'exists':
115-
requestBody.mustNot(esb.existsQuery(item.meta.key));
116-
break;
117-
case 'phrases':
118-
let negatedBody = esb.boolQuery();
119-
if (item.meta.value.indexOf(',') > -1) {
120-
const valueSplit = item.meta.value.split(', ');
121-
for (const [key, incr] of valueSplit.entries()) {
122-
negatedBody.should(esb.matchPhraseQuery(item.meta.key, incr));
123-
}
124-
} else {
125-
negatedBody.should(
126-
esb.matchPhraseQuery(item.meta.key, item.meta.value)
127-
);
128-
}
129-
negatedBody.minimumShouldMatch(1);
130-
requestBody.mustNot(negatedBody);
131-
break;
132-
case 'range':
133-
const builder = esb.rangeQuery(item.meta.key);
134-
if (item.meta.params.gte) builder.gte(item.meta.params.gte);
135-
if (item.meta.params.lte) builder.lte(item.meta.params.lte);
136-
if (item.meta.params.gt) builder.gt(item.meta.params.gt);
137-
if (item.meta.params.lt) builder.lt(item.meta.params.lt);
138-
requestBody.mustNot(builder);
139-
break;
140-
}
141-
break;
142-
}
143-
}
144-
}
145-
//search part
146-
let searchQuery = JSON.parse(filters)
147-
.query.query.replace(/ and /g, ' AND ')
148-
.replace(/ or /g, ' OR ')
149-
.replace(/ not /g, ' NOT ');
150-
if (searchQuery) {
151-
requestBody.must(esb.queryStringQuery(searchQuery));
152-
}
73+
export const buildRequestBody = (report: any, is_count: number) => {
74+
let esbBoolQuery = esb.boolQuery();
75+
const searchSourceJSON = report._source.searchSourceJSON;
76+
77+
const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query;
78+
const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter;
79+
const QueryFromSavedObject = buildOpenSearchQuery(
80+
undefined,
81+
savedObjectQuery,
82+
savedObjectFilter
83+
);
84+
// Add time range
15385
if (report._source.timeFieldName && report._source.timeFieldName.length > 0) {
154-
requestBody.must(
86+
esbBoolQuery.must(
15587
esb
15688
.rangeQuery(report._source.timeFieldName)
15789
.format('epoch_millis')
@@ -160,36 +92,58 @@ export const buildQuery = (report, is_count) => {
16092
);
16193
}
16294
if (is_count) {
163-
return esb.requestBodySearch().query(requestBody);
95+
return esb.requestBodySearch().query(esbBoolQuery);
16496
}
16597

166-
//Add the Sort to the query
167-
let reqBody = esb.requestBodySearch().query(requestBody).version(true);
98+
// Add sorting to the query
99+
let esbSearchQuery = esb
100+
.requestBodySearch()
101+
.query(esbBoolQuery)
102+
.version(true);
168103

169104
if (report._source.sorting.length > 0) {
170105
const sortings: Sort[] = report._source.sorting.map((element: string[]) => {
171106
return esb.sort(element[0], element[1]);
172107
});
173-
reqBody.sorts(sortings);
108+
esbSearchQuery.sorts(sortings);
174109
}
175110

176-
//get the selected fields only
111+
// add selected fields to query
177112
if (report._source.fields_exist) {
178-
reqBody.source({ includes: report._source.selectedFields });
113+
esbSearchQuery.source({ includes: report._source.selectedFields });
179114
}
180-
return reqBody;
115+
// Add a customizer to merge queries to generate request body
116+
let requestBody = _.mergeWith(
117+
{ query: QueryFromSavedObject },
118+
esbSearchQuery.toJSON(),
119+
(objValue, srcValue) => {
120+
if (_.isArray(objValue)) {
121+
return objValue.concat(srcValue);
122+
}
123+
}
124+
);
125+
126+
requestBody = addDocValueFields(report, requestBody);
127+
return requestBody;
181128
};
182129

183130
// Fetch the data from OpenSearch
184-
export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) => {
131+
export const getOpenSearchData = (
132+
arrayHits,
133+
report,
134+
params,
135+
dateFormat: string
136+
) => {
185137
let hits: any = [];
186138
for (let valueRes of arrayHits) {
187139
for (let data of valueRes.hits) {
188140
const fields = data.fields;
189141
// get all the fields of type date and format them to excel format
190142
for (let dateType of report._source.dateFields) {
191143
if (data._source[dateType]) {
192-
data._source[dateType] = moment(fields[dateType][0]).format(dateFormat);
144+
data._source[dateType] = moment(fields[dateType][0]).format(
145+
dateFormat
146+
);
193147
}
194148
}
195149
delete data['fields'];
@@ -274,3 +228,20 @@ function sanitize(doc: any) {
274228
}
275229
return doc;
276230
}
231+
232+
const addDocValueFields = (report: any, requestBody: any) => {
233+
const docValues = [];
234+
for (const dateType of report._source.dateFields) {
235+
docValues.push({
236+
field: dateType,
237+
format: 'date_hour_minute_second_fraction',
238+
});
239+
}
240+
// elastic-builder doesn't provide function to build docvalue_fields with format,
241+
// this is a workaround which appends docvalues field to the request body.
242+
requestBody = {
243+
...requestBody,
244+
docvalue_fields: docValues,
245+
};
246+
return requestBody;
247+
};

‎dashboards-reports/server/routes/utils/savedSearchReportHelper.ts‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
*/
2626

2727
import {
28-
buildQuery,
28+
buildRequestBody,
2929
convertToCSV,
3030
getOpenSearchData,
3131
getSelectedFields,
@@ -104,15 +104,15 @@ async function populateMetaData(
104104

105105
metaData.sorting = ssInfos._source.search.sort;
106106
metaData.type = ssInfos._source.type;
107-
metaData.filters =
107+
metaData.searchSourceJSON =
108108
ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON;
109109

110110
// Get the list of selected columns in the saved search.Otherwise select all the fields under the _source
111111
await getSelectedFields(ssInfos._source.search.columns);
112112

113113
// Get index name
114114
for (const item of ssInfos._source.references) {
115-
if (item.name === JSON.parse(metaData.filters).indexRefName) {
115+
if (item.name === JSON.parse(metaData.searchSourceJSON).indexRefName) {
116116
// Get index-pattern information
117117
const indexPattern = await callCluster(
118118
client,
@@ -163,7 +163,7 @@ async function generateReportData(
163163
return '';
164164
}
165165

166-
const reqBody = buildRequestBody(buildQuery(report, 0));
166+
const reqBody = buildRequestBody(report, 0);
167167
logger.info(
168168
`[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}`
169169
);
@@ -200,13 +200,13 @@ async function generateReportData(
200200

201201
// Build the OpenSearch Count query to count the size of result
202202
async function getOpenSearchDataSize() {
203-
const countReq = buildQuery(report, 1);
203+
const countReq = buildRequestBody(report, 1);
204204
return await callCluster(
205205
client,
206206
'count',
207207
{
208208
index: indexPattern,
209-
body: countReq.toJSON(),
209+
body: countReq,
210210
},
211211
isScheduledTask
212212
);
@@ -273,23 +273,6 @@ async function generateReportData(
273273
arrayHits.push(opensearchData.hits);
274274
}
275275

276-
function buildRequestBody(query: esb.RequestBodySearch) {
277-
const docvalues = [];
278-
for (const dateType of report._source.dateFields) {
279-
docvalues.push({
280-
field: dateType,
281-
format: 'date_hour_minute_second_fraction',
282-
});
283-
}
284-
285-
// elastic-builder doesn't provide function to build docvalue_fields with format,
286-
// this is a workaround which appends docvalues field to the request body.
287-
return {
288-
...query.toJSON(),
289-
docvalue_fields: docvalues,
290-
};
291-
}
292-
293276
// Parse OpenSearch data and convert to CSV
294277
async function convertOpenSearchDataToCsv() {
295278
const dataset: any = [];

0 commit comments

Comments
 (0)