Skip to content

Commit a35aac2

Browse files
BigFungerBigFunger
committed
Removed pipelines from ingest endpoint, and import csv wizard
1 parent 5bbe02e commit a35aac2

8 files changed

Lines changed: 10 additions & 159 deletions

File tree

‎src/core_plugins/kibana/public/management/sections/indices/add_data_steps/upload_data_step/upload_data_step.js‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,7 @@ modules.get('apps/management')
2727
location: 'Add Data'
2828
});
2929

30-
const usePipeline = !_.isEmpty(_.get(this.results, 'pipeline.processors'));
31-
ingest.uploadCSV(this.results.file, this.results.indexPattern.id, this.results.parseOptions.delimiter, usePipeline)
30+
ingest.uploadCSV(this.results.file, this.results.indexPattern.id, this.results.parseOptions.delimiter)
3231
.then(
3332
(res) => {
3433
this.created = 0;

‎src/core_plugins/kibana/public/management/sections/indices/filebeat/directives/filebeat_wizard.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import template from 'plugins/kibana/management/sections/indices/filebeat/direct
33
import IngestProvider from 'ui/ingest';
44
import 'plugins/kibana/management/sections/indices/add_data_steps/pattern_review_step';
55
import 'plugins/kibana/management/sections/indices/add_data_steps/paste_samples_step';
6-
import 'plugins/kibana/management/sections/indices/add_data_steps/pipeline_setup';
76
import 'plugins/kibana/management/sections/indices/add_data_steps/install_filebeat_step';
87
import '../../styles/_add_data_wizard.less';
98

‎src/core_plugins/kibana/server/lib/schemas/resources/ingest_config_schema.js‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
import Joi from 'joi';
22
import indexPatternSchema from './index_pattern_schema';
3-
import pipelineSchema from './pipeline_schema';
43

54
module.exports = Joi.object({
6-
index_pattern: indexPatternSchema.required(),
7-
pipeline: pipelineSchema
5+
index_pattern: indexPatternSchema.required()
86
});

‎src/core_plugins/kibana/server/routes/api/ingest/register_post.js‎

Lines changed: 1 addition & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import createMappingsFromPatternFields from '../../../lib/create_mappings_from_p
66
import initDefaultFieldProps from '../../../lib/init_default_field_props';
77
import {ingestToPattern, patternToIngest} from '../../../../common/lib/convert_pattern_and_ingest_name';
88
import { keysToCamelCaseShallow } from '../../../../common/lib/case_conversion';
9-
import ingestPipelineApiKibanaToEsConverter from '../../../lib/converters/ingest_pipeline_api_kibana_to_es_converter';
109

1110
export function registerPost(server) {
1211
const kibanaIndex = server.config().get('kibana.index');
@@ -25,7 +24,7 @@ export function registerPost(server) {
2524
},
2625
(patternDeletionError) => {
2726
throw new Error(
28-
`index-pattern ${indexPatternId} created successfully but index template or pipeline
27+
`index-pattern ${indexPatternId} created successfully but index template
2928
creation failed. Failed to rollback index-pattern creation, must delete manually.
3029
${patternDeletionError.toString()}
3130
${rootError.toString()}`
@@ -34,27 +33,6 @@ export function registerPost(server) {
3433
);
3534
}
3635

37-
function templateRollback(rootError, templateName, boundCallWithRequest) {
38-
const deleteParams = {
39-
name: templateName
40-
};
41-
42-
return boundCallWithRequest('indices.deleteTemplate', deleteParams)
43-
.then(
44-
() => {
45-
throw rootError;
46-
},
47-
(templateDeletionError) => {
48-
throw new Error(
49-
`index template ${templateName} created successfully but pipeline
50-
creation failed. Failed to rollback template creation, must delete manually.
51-
${templateDeletionError.toString()}
52-
${rootError.toString()}`
53-
);
54-
}
55-
);
56-
}
57-
5836
server.route({
5937
path: '/api/kibana/ingest',
6038
method: 'POST',
@@ -71,7 +49,6 @@ export function registerPost(server) {
7149
const indexPattern = keysToCamelCaseShallow(requestDocument.index_pattern);
7250
const indexPatternId = indexPattern.id;
7351
const ingestConfigName = patternToIngest(indexPatternId);
74-
const shouldCreatePipeline = !_.isEmpty(requestDocument.pipeline);
7552
delete indexPattern.id;
7653

7754
const mappings = createMappingsFromPatternFields(indexPattern.fields);
@@ -81,8 +58,6 @@ export function registerPost(server) {
8158
indexPattern.fields = JSON.stringify(indexPattern.fields);
8259
indexPattern.fieldFormatMap = JSON.stringify(indexPattern.fieldFormatMap);
8360

84-
const pipeline = ingestPipelineApiKibanaToEsConverter(requestDocument.pipeline);
85-
8661
// Set up call with request params
8762
const patternCreateParams = {
8863
index: kibanaIndex,
@@ -105,13 +80,6 @@ export function registerPost(server) {
10580
}
10681
};
10782

108-
const pipelineParams = {
109-
path: `/_ingest/pipeline/${ingestConfigName}`,
110-
method: 'PUT',
111-
body: pipeline
112-
};
113-
114-
11583
return boundCallWithRequest('indices.exists', {index: indexPatternId})
11684
.then((matchingIndices) => {
11785
if (matchingIndices) {
@@ -122,15 +90,6 @@ export function registerPost(server) {
12290
.then(() => {
12391
return boundCallWithRequest('indices.putTemplate', templateParams)
12492
.catch((templateError) => {return patternRollback(templateError, indexPatternId, boundCallWithRequest);});
125-
})
126-
.then((templateResponse) => {
127-
if (!shouldCreatePipeline) {
128-
return templateResponse;
129-
}
130-
131-
return boundCallWithRequest('transport.request', pipelineParams)
132-
.catch((pipelineError) => {return templateRollback(pipelineError, ingestConfigName, boundCallWithRequest);})
133-
.catch((templateRollbackError) => {return patternRollback(templateRollbackError, indexPatternId, boundCallWithRequest);});
13493
});
13594
})
13695
.then(

‎src/ui/public/ingest/__tests__/ingest.js‎

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -122,26 +122,26 @@ describe('Ingest Service', function () {
122122

123123
it('POSTs to the kibana _data endpoint with the correct params and the file attached as multipart/form-data', function () {
124124
$httpBackend
125-
.expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true', function (data) {
125+
.expectPOST('/api/kibana/foo/_data?csv_delimiter=;', function (data) {
126126
// The assertions we can do here are limited because of poor browser support for FormData methods
127127
return data instanceof FormData;
128128
})
129129
.respond('ok');
130130

131131
const file = new Blob(['foo,bar'], {type : 'text/csv'});
132132

133-
ingest.uploadCSV(file, 'foo', ';', true);
133+
ingest.uploadCSV(file, 'foo', ';');
134134
$httpBackend.flush();
135135
});
136136

137137
it('Returns error from the data API if there is one', function (done) {
138138
$httpBackend
139-
.expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true')
139+
.expectPOST('/api/kibana/foo/_data?csv_delimiter=;')
140140
.respond(404);
141141

142142
const file = new Blob(['foo,bar'], {type : 'text/csv'});
143143

144-
ingest.uploadCSV(file, 'foo', ';', true)
144+
ingest.uploadCSV(file, 'foo', ';')
145145
.then(
146146
() => {
147147
throw new Error('expected an error response');
@@ -156,35 +156,4 @@ describe('Ingest Service', function () {
156156
});
157157
});
158158

159-
describe('getProcessors', () => {
160-
161-
it('Calls the processors GET endpoint of the ingest API', function () {
162-
$httpBackend
163-
.expectGET('/api/kibana/ingest/processors')
164-
.respond('ok');
165-
166-
ingest.getProcessors();
167-
$httpBackend.flush();
168-
});
169-
170-
it('Throws user-friendly error when there is an error in the request', function (done) {
171-
$httpBackend
172-
.when('GET', '/api/kibana/ingest/processors')
173-
.respond(404);
174-
175-
ingest.getProcessors()
176-
.then(
177-
() => {
178-
throw new Error('expected an error response');
179-
},
180-
(error) => {
181-
expect(error.message).to.be('Error fetching enabled processors');
182-
done();
183-
});
184-
185-
$httpBackend.flush();
186-
});
187-
188-
});
189-
190159
});

‎src/ui/public/ingest/ingest.js‎

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,14 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i
99
const ingestAPIPrefix = chrome.addBasePath('/api/kibana/ingest');
1010
const refreshKibanaIndex = Private(RefreshKibanaIndexProvider);
1111

12-
this.save = function (indexPattern, pipeline) {
12+
this.save = function (indexPattern) {
1313
if (_.isEmpty(indexPattern)) {
1414
throw new Error('index pattern is required');
1515
}
1616

1717
const payload = {
1818
index_pattern: keysToSnakeCaseShallow(indexPattern)
1919
};
20-
if (!_.isEmpty(pipeline)) {
21-
payload.pipeline = _.map(pipeline, processor => keysToSnakeCaseShallow(processor));
22-
}
2320

2421
return $http.post(`${ingestAPIPrefix}`, payload)
2522
.then(() => {
@@ -44,39 +41,7 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i
4441
});
4542
};
4643

47-
this.simulate = function (pipeline) {
48-
function pack(pipeline) {
49-
const result = keysToSnakeCaseShallow(pipeline);
50-
result.processors = _.map(result.processors, processor => keysToSnakeCaseShallow(processor));
51-
52-
return result;
53-
}
54-
55-
function unpack(response) {
56-
const data = response.data.map(result => keysToCamelCaseShallow(result));
57-
return data;
58-
}
59-
60-
return $http.post(`${ingestAPIPrefix}/simulate`, pack(pipeline))
61-
.then(unpack)
62-
.catch(err => {
63-
return $q.reject(new Error('Error simulating pipeline'));
64-
});
65-
};
66-
67-
this.getProcessors = function () {
68-
function unpack(response) {
69-
return response.data;
70-
}
71-
72-
return $http.get(`${ingestAPIPrefix}/processors`)
73-
.then(unpack)
74-
.catch(err => {
75-
return $q.reject(new Error('Error fetching enabled processors'));
76-
});
77-
};
78-
79-
this.uploadCSV = function (file, indexPattern, delimiter, pipeline) {
44+
this.uploadCSV = function (file, indexPattern, delimiter) {
8045
if (_.isUndefined(file)) {
8146
throw new Error('file is required');
8247
}
@@ -91,9 +56,6 @@ export default function IngestProvider($rootScope, $http, config, $q, Private, i
9156
if (!_.isUndefined(delimiter)) {
9257
params.csv_delimiter = delimiter;
9358
}
94-
if (!_.isUndefined(pipeline)) {
95-
params.pipeline = pipeline;
96-
}
9759

9860
return $http.post(chrome.addBasePath(`/api/kibana/${indexPattern}/_data`), formData, {
9961
params: params,

‎test/unit/api/ingest/_post.js‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,6 @@ define(function (require) {
4343
// Fields must have a name and type
4444
request.post('/kibana/ingest')
4545
.send(_.set(createTestData(), 'index_pattern.fields', [{count: 0}]))
46-
.expect(400),
47-
48-
// should validate pipeline processors
49-
request.post('/kibana/ingest')
50-
.send(_.set(createTestData(), 'pipeline[0]', {bad: 'processor'}))
5146
.expect(400)
5247
]);
5348
});
@@ -165,30 +160,6 @@ define(function (require) {
165160
});
166161
});
167162

168-
bdd.it('should create a pipeline if one is included in the request', function () {
169-
return request.post('/kibana/ingest')
170-
.send(createTestData())
171-
.expect(204)
172-
.then(function () {
173-
return scenarioManager.client.transport.request({
174-
path: '_ingest/pipeline/kibana-logstash-*',
175-
method: 'GET'
176-
})
177-
.then(function (body) {
178-
expect(body).to.have.property('kibana-logstash-*');
179-
});
180-
});
181-
});
182-
183-
bdd.it('pipeline should be optional', function optionalPipeline() {
184-
const payload = createTestData();
185-
delete payload.pipeline;
186-
187-
return request.post('/kibana/ingest')
188-
.send(payload)
189-
.expect(204);
190-
});
191-
192163
bdd.it('should return 409 conflict when a pattern with the given ID already exists', function patternConflict() {
193164
return request.post('/kibana/ingest')
194165
.send(createTestData())

‎test/unit/api/ingest/data.js‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,6 @@ module.exports = function createTestData() {
2323
'type': 'geo_point'
2424
}
2525
]
26-
},
27-
pipeline: [{
28-
processor_id: 'processor1',
29-
type_id: 'set',
30-
target_field: 'foo',
31-
value: 'bar'
32-
}]
26+
}
3327
};
3428
};

0 commit comments

Comments
 (0)