Skip to content

Commit 2f1db15

Browse files
jen-huangelasticmachine
jen-huang
and
elasticmachine
committed
[Ingest Manager] Support updated package output structure (#69864)
* Update EPM package registry typings to reflect registry changes * Change `dataset.id` references to `dataset.name` * Fix RegistryStream * Fix packageToConfigDatasourceInputs service * Fix assignPackageStream service * Fix validateDatasource service * Fix configure data source components * Fix variable * Fix stream template installation * Add support for `input[].dataset.type` and change `stream.dataset` mapping to be object containing `name` with instead of just a string * Nest package information under `meta` in agent config yaml * Move `dataset.type` to stream level instead of input level * Make single call to fetch registry package information instead of doing it per stream * Fix type issues * Update endpoint test assertion Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
1 parent 3e48426 commit 2f1db15

27 files changed

Lines changed: 461 additions & 248 deletions

File tree

‎x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ const CONFIG_KEYS_ORDER = [
1616
'inputs',
1717
'enabled',
1818
'use_output',
19-
'package',
19+
'meta',
2020
'input',
2121
];
2222

‎x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
3939
{
4040
id: 'test-logs-foo',
4141
enabled: true,
42-
dataset: 'foo',
42+
dataset: { name: 'foo', type: 'logs' },
4343
vars: {
4444
fooVar: { value: 'foo-value' },
4545
fooVar2: { value: [1, 2] },
@@ -52,7 +52,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
5252
{
5353
id: 'test-logs-bar',
5454
enabled: true,
55-
dataset: 'bar',
55+
dataset: { name: 'bar', type: 'logs' },
5656
vars: {
5757
barVar: { value: 'bar-value' },
5858
barVar2: { value: [1, 2] },
@@ -101,23 +101,41 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
101101
});
102102

103103
it('returns agent inputs', () => {
104-
expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([
104+
expect(
105+
storedDatasourcesToAgentInputs([
106+
{
107+
...mockDatasource,
108+
package: {
109+
name: 'mock-package',
110+
title: 'Mock package',
111+
version: '0.0.0',
112+
},
113+
inputs: [mockInput],
114+
},
115+
])
116+
).toEqual([
105117
{
106118
id: 'some-uuid',
107119
name: 'mock-datasource',
108120
type: 'test-logs',
109121
dataset: { namespace: 'default' },
110122
use_output: 'default',
123+
meta: {
124+
package: {
125+
name: 'mock-package',
126+
version: '0.0.0',
127+
},
128+
},
111129
streams: [
112130
{
113131
id: 'test-logs-foo',
114-
dataset: { name: 'foo' },
132+
dataset: { name: 'foo', type: 'logs' },
115133
fooKey: 'fooValue1',
116134
fooKey2: ['fooValue2'],
117135
},
118136
{
119137
id: 'test-logs-bar',
120-
dataset: { name: 'bar' },
138+
dataset: { name: 'bar', type: 'logs' },
121139
},
122140
],
123141
},
@@ -147,7 +165,7 @@ describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
147165
streams: [
148166
{
149167
id: 'test-logs-foo',
150-
dataset: { name: 'foo' },
168+
dataset: { name: 'foo', type: 'logs' },
151169
fooKey: 'fooValue1',
152170
fooKey2: ['fooValue2'],
153171
},

‎x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@ export const storedDatasourcesToAgentInputs = (
2424
id: datasource.id || datasource.name,
2525
name: datasource.name,
2626
type: input.type,
27-
dataset: { namespace: datasource.namespace || 'default' },
27+
dataset: {
28+
namespace: datasource.namespace || 'default',
29+
},
2830
use_output: DEFAULT_OUTPUT.name,
2931
...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => {
3032
acc[key] = value;
@@ -35,7 +37,7 @@ export const storedDatasourcesToAgentInputs = (
3537
.map((stream) => {
3638
const fullStream: FullAgentConfigInputStream = {
3739
id: stream.id,
38-
dataset: { name: stream.dataset },
40+
dataset: stream.dataset,
3941
...stream.agent_stream,
4042
...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
4143
acc[key] = value;
@@ -50,9 +52,11 @@ export const storedDatasourcesToAgentInputs = (
5052
};
5153

5254
if (datasource.package) {
53-
fullInput.package = {
54-
name: datasource.package.name,
55-
version: datasource.package.version,
55+
fullInput.meta = {
56+
package: {
57+
name: datasource.package.name,
58+
version: datasource.package.version,
59+
},
5660
};
5761
}
5862

0 commit comments

Comments
 (0)