Skip to content

Commit 63dfc7d

Browse files
saarikabhasikibanamachine
saarikabhasi
and
kibanamachine
authored
[Serverless Search] Improve awareness on using pipelines (#182177)
## Summary - Adds a section for `Preprocessing data ` - [x] home page - [x] new connector ->connect to index tab - Remove `Transform and enrich your data` from home page ### Screen Recording https://github.com/elastic/kibana/assets/55930906/47820f68-a726-4ad5-83ab-2fabcb899c91 ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
1 parent 31e8c54 commit 63dfc7d

24 files changed

Lines changed: 483 additions & 89 deletions

File tree

‎packages/kbn-doc-links/src/get_doc_links.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -791,6 +791,11 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D
791791
csvPipelines: `${ELASTIC_WEBSITE_URL}guide/en/ecs/${ECS_VERSION}/ecs-converting.html`,
792792
pipelineFailure: `${ELASTICSEARCH_DOCS}ingest.html#handling-pipeline-failures`,
793793
processors: `${ELASTICSEARCH_DOCS}processors.html`,
794+
arrayOrJson: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-array-json-handling`,
795+
dataEnrichment: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-data-enrichment`,
796+
dataFiltering: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-data-filtering`,
797+
dataTransformation: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-data-transformation`,
798+
pipelineHandling: `${ELASTICSEARCH_DOCS}processors.html#ingest-process-category-pipeline-handling`,
794799
remove: `${ELASTICSEARCH_DOCS}remove-processor.html`,
795800
rename: `${ELASTICSEARCH_DOCS}rename-processor.html`,
796801
script: `${ELASTICSEARCH_DOCS}script-processor.html`,

‎packages/kbn-search-api-panels/components/preprocess_data.tsx‎

Lines changed: 245 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
import {
9+
EuiFlexGroup,
10+
EuiFlexItem,
11+
EuiSpacer,
12+
EuiText,
13+
EuiThemeProvider,
14+
EuiTitle,
15+
EuiPanel,
16+
EuiLink,
17+
EuiFlexGrid,
18+
EuiImage,
19+
} from '@elastic/eui';
20+
import { i18n } from '@kbn/i18n';
21+
import { euiThemeVars } from '@kbn/ui-theme';
22+
import React from 'react';
23+
24+
export const PreprocessDataPanel: React.FC<{
25+
docLinks: {
26+
arrayOrJson: string;
27+
dataEnrichment: string;
28+
dataFiltering: string;
29+
dataTransformation: string;
30+
pipelineHandling: string;
31+
};
32+
images: {
33+
dataEnrichment: string;
34+
dataTransformation: string;
35+
dataFiltering: string;
36+
pipelineHandling: string;
37+
arrayHandling: string;
38+
};
39+
}> = ({ docLinks, images }) => {
40+
const color = euiThemeVars.euiColorVis1_behindText;
41+
return (
42+
<EuiThemeProvider colorMode="dark">
43+
<EuiPanel paddingSize="xl">
44+
<EuiFlexGrid direction="column" columns={2} gutterSize="xl">
45+
<EuiFlexItem>
46+
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
47+
<EuiFlexItem grow={false}>
48+
<EuiImage alt="" src={images.dataEnrichment} color={color} />
49+
</EuiFlexItem>
50+
<EuiFlexItem>
51+
<EuiTitle size="s">
52+
<h3>
53+
{i18n.translate(
54+
'searchApiPanels.preprocessData.overview.dataEnrichment.title',
55+
{
56+
defaultMessage: 'Data enrichment',
57+
}
58+
)}
59+
</h3>
60+
</EuiTitle>
61+
<EuiSpacer size="s" />
62+
<EuiText size="s">
63+
<p>
64+
{i18n.translate(
65+
'searchApiPanels.preprocessData.overview.dataEnrichment.description',
66+
{
67+
defaultMessage:
68+
'Add information from external sources or apply transformations to your documents for more contextual, insightful search.',
69+
}
70+
)}
71+
</p>
72+
</EuiText>
73+
<EuiSpacer size="s" />
74+
<EuiText>
75+
<p>
76+
<EuiLink href={docLinks.dataEnrichment} target="_blank">
77+
{i18n.translate(
78+
'searchApiPanels.preprocessData.overview.dataEnrichment.learnMore',
79+
{
80+
defaultMessage: 'Learn more',
81+
}
82+
)}
83+
</EuiLink>
84+
</p>
85+
</EuiText>
86+
</EuiFlexItem>
87+
</EuiFlexGroup>
88+
</EuiFlexItem>
89+
<EuiFlexItem>
90+
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
91+
<EuiFlexItem grow={false}>
92+
<EuiImage alt="" src={images.dataFiltering} color={color} />
93+
</EuiFlexItem>
94+
<EuiFlexItem>
95+
<EuiTitle size="s">
96+
<h3>
97+
{i18n.translate('searchApiPanels.pipeline.overview.dataFiltering.title', {
98+
defaultMessage: 'Data filtering',
99+
})}
100+
</h3>
101+
</EuiTitle>
102+
<EuiSpacer size="s" />
103+
<EuiText size="s">
104+
{i18n.translate('searchApiPanels.pipeline.overview.dataFiltering.description', {
105+
defaultMessage:
106+
'Remove specific fields from documents before indexing, to exclude unnecessary or sensitive information.',
107+
})}
108+
</EuiText>
109+
<EuiSpacer size="s" />
110+
<EuiText>
111+
<p>
112+
<EuiLink href={docLinks.dataFiltering} target="_blank">
113+
{i18n.translate(
114+
'searchApiPanels.preprocessData.overview.dataFiltering.learnMore',
115+
{
116+
defaultMessage: 'Learn more',
117+
}
118+
)}
119+
</EuiLink>
120+
</p>
121+
</EuiText>
122+
</EuiFlexItem>
123+
</EuiFlexGroup>
124+
</EuiFlexItem>
125+
<EuiFlexItem>
126+
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
127+
<EuiFlexItem grow={false}>
128+
<EuiImage alt="" src={images.arrayHandling} color={color} />
129+
</EuiFlexItem>
130+
<EuiFlexItem>
131+
<EuiTitle size="s">
132+
<h3>
133+
{i18n.translate('searchApiPanels.pipeline.overview.arrayJsonHandling.title', {
134+
defaultMessage: 'Array/JSON handling',
135+
})}
136+
</h3>
137+
</EuiTitle>
138+
<EuiSpacer size="s" />
139+
<EuiText size="s">
140+
{i18n.translate(
141+
'searchApiPanels.pipeline.overview.arrayJsonHandling.description',
142+
{
143+
defaultMessage: 'Run batch processors, parse JSON data and sort elements.',
144+
}
145+
)}
146+
</EuiText>
147+
<EuiSpacer size="s" />
148+
<EuiText>
149+
<p>
150+
<EuiLink href={docLinks.arrayOrJson} target="_blank">
151+
{i18n.translate(
152+
'searchApiPanels.preprocessData.overview.arrayJsonHandling.learnMore',
153+
{
154+
defaultMessage: 'Learn more',
155+
}
156+
)}
157+
</EuiLink>
158+
</p>
159+
</EuiText>
160+
</EuiFlexItem>
161+
</EuiFlexGroup>
162+
</EuiFlexItem>
163+
<EuiFlexItem>
164+
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
165+
<EuiFlexItem grow={false}>
166+
<EuiImage alt="" src={images.dataTransformation} color={color} />
167+
</EuiFlexItem>
168+
<EuiFlexItem>
169+
<EuiTitle size="s">
170+
<h3>
171+
{i18n.translate('searchApiPanels.pipeline.overview.dataTransformation.title', {
172+
defaultMessage: 'Data transformation',
173+
})}
174+
</h3>
175+
</EuiTitle>
176+
<EuiSpacer size="s" />
177+
<EuiText size="s">
178+
{i18n.translate(
179+
'searchApiPanels.pipeline.overview.dataTransformation.description',
180+
{
181+
defaultMessage:
182+
'Parse information from your documents to ensure they conform to a standardized format.',
183+
}
184+
)}
185+
</EuiText>
186+
<EuiSpacer size="s" />
187+
<EuiText size="s">
188+
<p>
189+
<EuiLink href={docLinks.dataTransformation} target="_blank">
190+
{i18n.translate(
191+
'searchApiPanels.preprocessData.overview.dataTransformation.learnMore',
192+
{
193+
defaultMessage: 'Learn more',
194+
}
195+
)}
196+
</EuiLink>
197+
</p>
198+
</EuiText>
199+
</EuiFlexItem>
200+
</EuiFlexGroup>
201+
</EuiFlexItem>
202+
<EuiFlexItem>
203+
<EuiFlexGroup alignItems="flexStart" justifyContent="flexStart">
204+
<EuiFlexItem grow={false}>
205+
<EuiImage alt="" src={images.pipelineHandling} color={color} />
206+
</EuiFlexItem>
207+
<EuiFlexItem>
208+
<EuiTitle size="s">
209+
<h3>
210+
{i18n.translate('searchApiPanels.pipeline.overview.pipelineHandling.title', {
211+
defaultMessage: 'Pipeline handling',
212+
})}
213+
</h3>
214+
</EuiTitle>
215+
216+
<EuiText size="s">
217+
{i18n.translate(
218+
'searchApiPanels.pipeline.overview.pipelineHandling.description',
219+
{
220+
defaultMessage:
221+
'Handle error exceptions, execute another pipeline, or reroute documents to another index',
222+
}
223+
)}
224+
</EuiText>
225+
<EuiSpacer size="s" />
226+
<EuiText>
227+
<p>
228+
<EuiLink href={docLinks.pipelineHandling} target="_blank">
229+
{i18n.translate(
230+
'searchApiPanels.preprocessData.overview.pipelineHandling.learnMore',
231+
{
232+
defaultMessage: 'Learn more',
233+
}
234+
)}
235+
</EuiLink>
236+
</p>
237+
</EuiText>
238+
</EuiFlexItem>
239+
</EuiFlexGroup>
240+
</EuiFlexItem>
241+
</EuiFlexGrid>
242+
</EuiPanel>
243+
</EuiThemeProvider>
244+
);
245+
};

‎packages/kbn-search-api-panels/index.tsx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export * from './components/pipeline_panel';
2222
export * from './components/select_client';
2323
export * from './components/try_in_console_button';
2424
export * from './components/install_client';
25+
export * from './components/preprocess_data';
2526

2627
export * from './types';
2728
export * from './utils';

‎packages/kbn-search-api-panels/tsconfig.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"@kbn/share-plugin",
2222
"@kbn/i18n-react",
2323
"@kbn/security-plugin",
24-
"@kbn/console-plugin"
24+
"@kbn/console-plugin",
25+
"@kbn/ui-theme"
2526
]
2627
}

‎x-pack/plugins/serverless_search/common/doc_links.ts‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,14 @@ class ESDocLinks {
5252
public gettingStartedSearch: string = '';
5353
public gettingStartedExplore: string = '';
5454

55+
// Ingest processor
56+
public dataEnrichment: string = '';
57+
public dataFiltering: string = '';
58+
public arrayOrJson: string = '';
59+
public dataTransformation: string = '';
60+
public pipelineHandling: string = '';
61+
public pipelines: string = '';
62+
5563
constructor() {}
5664

5765
setDocLinks(newDocLinks: DocLinks) {
@@ -94,7 +102,7 @@ class ESDocLinks {
94102
this.pythonApiReference = newDocLinks.serverlessClients.pythonGettingStarted;
95103
this.pythonBasicConfig = newDocLinks.serverlessClients.pythonGettingStarted;
96104
this.pythonClient = newDocLinks.serverlessClients.pythonGettingStarted;
97-
// Python
105+
// Ruby
98106
this.rubyBasicConfig = newDocLinks.serverlessClients.rubyGettingStarted;
99107
this.rubyExamples = newDocLinks.serverlessClients.rubyApiReference;
100108
this.rubyClient = newDocLinks.serverlessClients.rubyGettingStarted;
@@ -103,6 +111,14 @@ class ESDocLinks {
103111
this.gettingStartedIngest = newDocLinks.serverlessSearch.gettingStartedIngest;
104112
this.gettingStartedSearch = newDocLinks.serverlessSearch.gettingStartedSearch;
105113
this.gettingStartedExplore = newDocLinks.serverlessSearch.gettingStartedExplore;
114+
115+
// Ingest processor
116+
this.dataEnrichment = newDocLinks.ingest.dataEnrichment;
117+
this.dataFiltering = newDocLinks.ingest.dataFiltering;
118+
this.arrayOrJson = newDocLinks.ingest.arrayOrJson;
119+
this.dataTransformation = newDocLinks.ingest.dataTransformation;
120+
this.pipelineHandling = newDocLinks.ingest.pipelineHandling;
121+
this.pipelines = newDocLinks.ingest.pipelines;
106122
}
107123
}
108124

‎x-pack/plugins/serverless_search/public/application/components/connectors/connector_config/connector_index_name.tsx‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,31 @@
55
* 2.0.
66
*/
77

8-
import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui';
8+
import {
9+
EuiButton,
10+
EuiFlexGroup,
11+
EuiFlexItem,
12+
EuiLink,
13+
EuiPanel,
14+
EuiSpacer,
15+
EuiText,
16+
EuiTitle,
17+
EuiCode,
18+
} from '@elastic/eui';
919
import { i18n } from '@kbn/i18n';
1020
import { Connector, ConnectorStatus } from '@kbn/search-connectors';
1121
import React, { useState } from 'react';
1222
import { useQueryClient, useMutation } from '@tanstack/react-query';
23+
import { FormattedMessage } from '@kbn/i18n-react';
1324
import { isValidIndexName } from '../../../../utils/validate_index_name';
1425
import { SAVE_LABEL } from '../../../../../common/i18n_string';
1526
import { useConnector } from '../../../hooks/api/use_connector';
1627
import { useKibanaServices } from '../../../hooks/use_kibana';
1728
import { ApiKeyPanel } from './api_key_panel';
1829
import { ConnectorIndexNameForm } from './connector_index_name_form';
1930
import { SyncScheduledCallOut } from './sync_scheduled_callout';
20-
31+
import { docLinks } from '../../../../../common/doc_links';
32+
import { DEFAULT_INGESTION_PIPELINE } from '../../../constants';
2133
interface ConnectorIndexNameProps {
2234
connector: Connector;
2335
}
@@ -80,6 +92,51 @@ export const ConnectorIndexName: React.FC<ConnectorIndexNameProps> = ({ connecto
8092
onChange={(name) => setNewIndexname(name)}
8193
/>
8294
<EuiSpacer />
95+
<EuiPanel hasBorder>
96+
<EuiFlexGroup direction="column" justifyContent="spaceBetween">
97+
<EuiFlexItem grow={false}>
98+
<EuiTitle size="s">
99+
<h3>
100+
{i18n.translate('xpack.serverlessSearch.connectors.config.preprocessData.title', {
101+
defaultMessage: 'Preprocess your data',
102+
})}
103+
</h3>
104+
</EuiTitle>
105+
</EuiFlexItem>
106+
<EuiFlexItem grow={false}>
107+
<EuiText>
108+
<p>
109+
<FormattedMessage
110+
id="xpack.serverlessSearch.connectors.config.preprocessData.description"
111+
defaultMessage="Use ingest pipelines to preprocess data before indexing into Elasticsearch. Note that connector clients use the {clientIngestionPipeline} pipeline for preprocessing."
112+
values={{
113+
clientIngestionPipeline: <EuiCode>{DEFAULT_INGESTION_PIPELINE}</EuiCode>,
114+
}}
115+
/>
116+
</p>
117+
</EuiText>
118+
</EuiFlexItem>
119+
<EuiFlexItem>
120+
<EuiText>
121+
<p>
122+
<EuiLink
123+
data-test-subj="serverlessSearchConnectorIndexNameLearnMoreLink"
124+
href={docLinks.pipelines}
125+
target="_blank"
126+
>
127+
{i18n.translate(
128+
'xpack.serverlessSearch.connectors.config.preprocessDataTitle.learnMore',
129+
{
130+
defaultMessage: 'Learn More',
131+
}
132+
)}
133+
</EuiLink>
134+
</p>
135+
</EuiText>
136+
</EuiFlexItem>
137+
</EuiFlexGroup>
138+
</EuiPanel>
139+
<EuiSpacer />
83140
<ApiKeyPanel connector={connector} />
84141
<EuiSpacer />
85142
<EuiFlexGroup>

0 commit comments

Comments
 (0)