Skip to content

Commit 3a6cb1b

Browse files
angoraycangorayc
committed
[SIEM] move away from Joi for importing/exporting timeline (#62125)
* move away from joi * update schema for filterQuery * fix types * update schemas * remove boom * remove redundant params * reuse utils from case * update schemas for query params and body * fix types * update validation schema * fix unit test * update description for test cases * remove import from case * lifting common libs * fix dependency * lifting validation builder function * add unit test * fix for code review * reve comments * rename common utils * fix types
1 parent a50a867 commit 3a6cb1b

21 files changed

Lines changed: 378 additions & 398 deletions

‎x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/import_rules_route.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@ import {
2626
buildSiemResponse,
2727
validateLicenseForRuleType,
2828
} from '../utils';
29-
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
3029
import { ImportRuleAlertRest } from '../../types';
3130
import { patchRules } from '../../rules/patch_rules';
3231
import { importRulesQuerySchema, importRulesPayloadSchema } from '../schemas/import_rules_schema';
3332
import { ImportRulesSchema, importRulesSchema } from '../schemas/response/import_rules_schema';
3433
import { getTupleDuplicateErrorsAndUniqueRules } from './utils';
3534
import { validate } from './validate';
35+
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
3636

3737
type PromiseFromStreams = ImportRuleAlertRest | Error;
3838

‎x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.test.ts‎

Lines changed: 1 addition & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import {
1212
transformTags,
1313
getIdBulkError,
1414
transformOrBulkError,
15-
transformDataToNdjson,
1615
transformAlertsToRules,
1716
transformOrImportError,
1817
getDuplicates,
@@ -22,14 +21,13 @@ import { getResult } from '../__mocks__/request_responses';
2221
import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
2322
import { ImportRuleAlertRest, RuleAlertParamsRest, RuleTypeParams } from '../../types';
2423
import { BulkError, ImportSuccessError } from '../utils';
25-
import { sampleRule } from '../../signals/__mocks__/es_results';
2624
import { getSimpleRule, getOutputRuleAlertForRest } from '../__mocks__/utils';
27-
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
2825
import { createPromiseFromStreams } from '../../../../../../../../../src/legacy/utils/streams';
2926
import { PartialAlert } from '../../../../../../../../plugins/alerting/server';
3027
import { SanitizedAlert } from '../../../../../../../../plugins/alerting/server/types';
3128
import { RuleAlertType } from '../../rules/types';
3229
import { setFeatureFlagsForTestsOnly, unSetFeatureFlagsForTestsOnly } from '../../feature_flags';
30+
import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson';
3331

3432
type PromiseFromStreams = ImportRuleAlertRest | Error;
3533

@@ -396,47 +394,6 @@ describe('utils', () => {
396394
});
397395
});
398396

399-
describe('transformDataToNdjson', () => {
400-
test('if rules are empty it returns an empty string', () => {
401-
const ruleNdjson = transformDataToNdjson([]);
402-
expect(ruleNdjson).toEqual('');
403-
});
404-
405-
test('single rule will transform with new line ending character for ndjson', () => {
406-
const rule = sampleRule();
407-
const ruleNdjson = transformDataToNdjson([rule]);
408-
expect(ruleNdjson.endsWith('\n')).toBe(true);
409-
});
410-
411-
test('multiple rules will transform with two new line ending characters for ndjson', () => {
412-
const result1 = sampleRule();
413-
const result2 = sampleRule();
414-
result2.id = 'some other id';
415-
result2.rule_id = 'some other id';
416-
result2.name = 'Some other rule';
417-
418-
const ruleNdjson = transformDataToNdjson([result1, result2]);
419-
// this is how we count characters in JavaScript :-)
420-
const count = ruleNdjson.split('\n').length - 1;
421-
expect(count).toBe(2);
422-
});
423-
424-
test('you can parse two rules back out without errors', () => {
425-
const result1 = sampleRule();
426-
const result2 = sampleRule();
427-
result2.id = 'some other id';
428-
result2.rule_id = 'some other id';
429-
result2.name = 'Some other rule';
430-
431-
const ruleNdjson = transformDataToNdjson([result1, result2]);
432-
const ruleStrings = ruleNdjson.split('\n');
433-
const reParsed1 = JSON.parse(ruleStrings[0]);
434-
const reParsed2 = JSON.parse(ruleStrings[1]);
435-
expect(reParsed1).toEqual(result1);
436-
expect(reParsed2).toEqual(result2);
437-
});
438-
});
439-
440397
describe('transformAlertsToRules', () => {
441398
test('given an empty array returns an empty array', () => {
442399
expect(transformAlertsToRules([])).toEqual([]);

‎x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -152,15 +152,6 @@ export const transformAlertToRule = (
152152
});
153153
};
154154

155-
export const transformDataToNdjson = (data: unknown[]): string => {
156-
if (data.length !== 0) {
157-
const dataString = data.map(rule => JSON.stringify(rule)).join('\n');
158-
return `${dataString}\n`;
159-
} else {
160-
return '';
161-
}
162-
};
163-
164155
export const transformAlertsToRules = (
165156
alerts: RuleAlertType[]
166157
): Array<Partial<OutputRuleAlertRest>> => {

‎x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/create_rules_stream_from_ndjson.ts‎

Lines changed: 5 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,39 +4,19 @@
44
* you may not use this file except in compliance with the Elastic License.
55
*/
66
import { Transform } from 'stream';
7-
import { has, isString } from 'lodash/fp';
87
import { ImportRuleAlertRest } from '../types';
98
import {
109
createSplitStream,
1110
createMapStream,
12-
createFilterStream,
1311
createConcatStream,
1412
} from '../../../../../../../../src/legacy/utils/streams';
1513
import { importRulesSchema } from '../routes/schemas/import_rules_schema';
1614
import { BadRequestError } from '../errors/bad_request_error';
17-
18-
export interface RulesObjectsExportResultDetails {
19-
/** number of successfully exported objects */
20-
exportedCount: number;
21-
}
22-
23-
export const parseNdjsonStrings = (): Transform => {
24-
return createMapStream((ndJsonStr: string) => {
25-
if (isString(ndJsonStr) && ndJsonStr.trim() !== '') {
26-
try {
27-
return JSON.parse(ndJsonStr);
28-
} catch (err) {
29-
return err;
30-
}
31-
}
32-
});
33-
};
34-
35-
export const filterExportedCounts = (): Transform => {
36-
return createFilterStream<ImportRuleAlertRest | RulesObjectsExportResultDetails>(
37-
obj => obj != null && !has('exported_count', obj)
38-
);
39-
};
15+
import {
16+
parseNdjsonStrings,
17+
filterExportedCounts,
18+
createLimitStream,
19+
} from '../../../utils/read_stream/create_stream_from_ndjson';
4020

4121
export const validateRules = (): Transform => {
4222
return createMapStream((obj: ImportRuleAlertRest) => {
@@ -53,21 +33,6 @@ export const validateRules = (): Transform => {
5333
});
5434
};
5535

56-
// Adaptation from: saved_objects/import/create_limit_stream.ts
57-
export const createLimitStream = (limit: number): Transform => {
58-
let counter = 0;
59-
return new Transform({
60-
objectMode: true,
61-
async transform(obj, _, done) {
62-
if (counter >= limit) {
63-
return done(new Error(`Can't import more than ${limit} rules`));
64-
}
65-
counter++;
66-
done(undefined, obj);
67-
},
68-
});
69-
};
70-
7136
// TODO: Capture both the line number and the rule_id if you have that information for the error message
7237
// eventually and then pass it down so we can give error messages on the line number
7338

‎x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_all.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
import { AlertsClient } from '../../../../../../../plugins/alerting/server';
88
import { getNonPackagedRules } from './get_existing_prepackaged_rules';
99
import { getExportDetailsNdjson } from './get_export_details_ndjson';
10-
import { transformAlertsToRules, transformDataToNdjson } from '../routes/rules/utils';
10+
import { transformAlertsToRules } from '../routes/rules/utils';
11+
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
1112

1213
export const getExportAll = async (
1314
alertsClient: AlertsClient

‎x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/get_export_by_object_ids.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ import { AlertsClient } from '../../../../../../../plugins/alerting/server';
88
import { getExportDetailsNdjson } from './get_export_details_ndjson';
99
import { isAlertType } from '../rules/types';
1010
import { readRules } from './read_rules';
11-
import { transformDataToNdjson, transformAlertToRule } from '../routes/rules/utils';
11+
import { transformAlertToRule } from '../routes/rules/utils';
1212
import { OutputRuleAlertRest } from '../types';
13+
import { transformDataToNdjson } from '../../../utils/read_stream/create_stream_from_ndjson';
1314

1415
interface ExportSuccesRule {
1516
statusCode: 200;

‎x-pack/legacy/plugins/siem/server/lib/timeline/create_timelines_stream_from_ndjson.ts‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,12 @@
33
* or more contributor license agreements. Licensed under the Elastic License;
44
* you may not use this file except in compliance with the Elastic License.
55
*/
6-
6+
import * as rt from 'io-ts';
77
import { Transform } from 'stream';
8+
import { pipe } from 'fp-ts/lib/pipeable';
9+
import { fold } from 'fp-ts/lib/Either';
10+
import { failure } from 'io-ts/lib/PathReporter';
11+
import { identity } from 'fp-ts/lib/function';
812
import {
913
createConcatStream,
1014
createSplitStream,
@@ -14,26 +18,28 @@ import {
1418
parseNdjsonStrings,
1519
filterExportedCounts,
1620
createLimitStream,
17-
} from '../detection_engine/rules/create_rules_stream_from_ndjson';
18-
import { importTimelinesSchema } from './routes/schemas/import_timelines_schema';
19-
import { BadRequestError } from '../detection_engine/errors/bad_request_error';
21+
} from '../../utils/read_stream/create_stream_from_ndjson';
22+
2023
import { ImportTimelineResponse } from './routes/utils/import_timelines';
24+
import { ImportTimelinesSchemaRt } from './routes/schemas/import_timelines_schema';
25+
26+
type ErrorFactory = (message: string) => Error;
2127

22-
export const validateTimelines = (): Transform => {
23-
return createMapStream((obj: ImportTimelineResponse) => {
24-
if (!(obj instanceof Error)) {
25-
const validated = importTimelinesSchema.validate(obj);
26-
if (validated.error != null) {
27-
return new BadRequestError(validated.error.message);
28-
} else {
29-
return validated.value;
30-
}
31-
} else {
32-
return obj;
33-
}
34-
});
28+
export const createPlainError = (message: string) => new Error(message);
29+
30+
export const throwErrors = (createError: ErrorFactory) => (errors: rt.Errors) => {
31+
throw createError(failure(errors).join('\n'));
3532
};
3633

34+
export const decodeOrThrow = <A, O, I>(
35+
runtimeType: rt.Type<A, O, I>,
36+
createError: ErrorFactory = createPlainError
37+
) => (inputValue: I) =>
38+
pipe(runtimeType.decode(inputValue), fold(throwErrors(createError), identity));
39+
40+
export const validateTimelines = (): Transform =>
41+
createMapStream((obj: ImportTimelineResponse) => decodeOrThrow(ImportTimelinesSchemaRt)(obj));
42+
3743
export const createTimelinesStreamFromNdJson = (ruleLimit: number) => {
3844
return [
3945
createSplitStream('\n'),

‎x-pack/legacy/plugins/siem/server/lib/timeline/routes/__mocks__/request_responses.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,16 @@
66

77
import { TIMELINE_EXPORT_URL, TIMELINE_IMPORT_URL } from '../../../../../common/constants';
88
import { requestMock } from '../../../detection_engine/routes/__mocks__';
9-
9+
import stream from 'stream';
10+
const readable = new stream.Readable();
1011
export const getExportTimelinesRequest = () =>
1112
requestMock.create({
1213
method: 'get',
1314
path: TIMELINE_EXPORT_URL,
15+
query: {
16+
file_name: 'mock_export_timeline.ndjson',
17+
exclude_export_details: 'false',
18+
},
1419
body: {
1520
ids: ['f0e58720-57b6-11ea-b88d-3f1a31716be8', '890b8ae0-57df-11ea-a7c9-3976b7f1cb37'],
1621
},
@@ -22,7 +27,7 @@ export const getImportTimelinesRequest = (filename?: string) =>
2227
path: TIMELINE_IMPORT_URL,
2328
query: { overwrite: false },
2429
body: {
25-
file: { hapi: { filename: filename ?? 'filename.ndjson' } },
30+
file: { ...readable, hapi: { filename: filename ?? 'filename.ndjson' } },
2631
},
2732
});
2833

‎x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.test.ts‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,15 +83,34 @@ describe('export timelines', () => {
8383
});
8484

8585
describe('request validation', () => {
86-
test('disallows singular id query param', async () => {
86+
test('return validation error for request body', async () => {
8787
const request = requestMock.create({
8888
method: 'get',
8989
path: TIMELINE_EXPORT_URL,
9090
body: { id: 'someId' },
9191
});
9292
const result = server.validate(request);
9393

94-
expect(result.badRequest).toHaveBeenCalledWith('"id" is not allowed');
94+
expect(result.badRequest.mock.calls[0][0]).toEqual(
95+
'Invalid value undefined supplied to : { ids: Array<string> }/ids: Array<string>'
96+
);
97+
});
98+
99+
test('return validation error for request params', async () => {
100+
const request = requestMock.create({
101+
method: 'get',
102+
path: TIMELINE_EXPORT_URL,
103+
body: { id: 'someId' },
104+
});
105+
const result = server.validate(request);
106+
107+
expect(result.badRequest.mock.calls[1][0]).toEqual(
108+
[
109+
'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/file_name: string',
110+
'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/0: "true"',
111+
'Invalid value undefined supplied to : { file_name: string, exclude_export_details: ("true" | "false") }/exclude_export_details: ("true" | "false")/1: "false"',
112+
].join('\n')
113+
);
95114
});
96115
});
97116
});

‎x-pack/legacy/plugins/siem/server/lib/timeline/routes/export_timelines_route.ts‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,24 @@
77
import { set as _set } from 'lodash/fp';
88
import { IRouter } from '../../../../../../../../src/core/server';
99
import { LegacyServices } from '../../../types';
10-
import { ExportTimelineRequestParams } from '../types';
1110

12-
import {
13-
transformError,
14-
buildRouteValidation,
15-
buildSiemResponse,
16-
} from '../../detection_engine/routes/utils';
11+
import { transformError, buildSiemResponse } from '../../detection_engine/routes/utils';
1712
import { TIMELINE_EXPORT_URL } from '../../../../common/constants';
1813

14+
import { getExportTimelineByObjectIds } from './utils/export_timelines';
1915
import {
20-
exportTimelinesSchema,
2116
exportTimelinesQuerySchema,
17+
exportTimelinesRequestBodySchema,
2218
} from './schemas/export_timelines_schema';
23-
24-
import { getExportTimelineByObjectIds } from './utils/export_timelines';
19+
import { buildRouteValidation } from '../../../utils/build_validation/route_validation';
2520

2621
export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['config']) => {
2722
router.post(
2823
{
2924
path: TIMELINE_EXPORT_URL,
3025
validate: {
31-
query: buildRouteValidation<ExportTimelineRequestParams['query']>(
32-
exportTimelinesQuerySchema
33-
),
34-
body: buildRouteValidation<ExportTimelineRequestParams['body']>(exportTimelinesSchema),
26+
query: buildRouteValidation(exportTimelinesQuerySchema),
27+
body: buildRouteValidation(exportTimelinesRequestBodySchema),
3528
},
3629
options: {
3730
tags: ['access:siem'],
@@ -42,6 +35,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co
4235
const siemResponse = buildSiemResponse(response);
4336
const savedObjectsClient = context.core.savedObjects.client;
4437
const exportSizeLimit = config().get<number>('savedObjects.maxImportExportSize');
38+
4539
if (request.body?.ids != null && request.body.ids.length > exportSizeLimit) {
4640
return siemResponse.error({
4741
statusCode: 400,
@@ -51,7 +45,7 @@ export const exportTimelinesRoute = (router: IRouter, config: LegacyServices['co
5145

5246
const responseBody = await getExportTimelineByObjectIds({
5347
client: savedObjectsClient,
54-
request,
48+
ids: request.body.ids,
5549
});
5650

5751
return response.ok({

0 commit comments

Comments
 (0)