Skip to content

Commit 08f70b7

Browse files
bhapaselasticmachinekibanamachine
authored
[Automatic Import] Introduce support for structured logs (#191749)
## Summary This PR introduces `KVGraph` that is used to support `structured` log samples. Examples of structured log samples would be: ``` <134>1 1639132850.430422377 AP1 events type=disassociation radio='1' vap='1' client_mac='B0:A4:60:9B:3B:A6' channel='100' reason='1' instigator='2' duration='223.031691642' auth_neg_dur='0.005054229' last_auth_ago='223.020414600' is_wpa='1' full_conn='0.384002374' ip_resp='0.384002374' ip_src='10.197.39.50' http_resp='0.647356228' arp_resp='0.013562625' arp_src='10.197.39.50' dns_server='10.128.128.128' dns_req_rtt='0.023370084' dns_resp='0.263616104' dhcp_lease_completed='0.009196083' dhcp_server='10.128.128.128' dhcp_server_mac='E0:CB:BC:31:23:60' dhcp_resp='0.009196083' aid='977866432' ``` Currently the tests prove that it works best with the log samples adhering to `RFC5424` and `RFC3164`. The Graph shall be improved to work with `Custom Formats` going forward. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### For maintainers - [ ] This was checked for breaking API changes and was [labeled appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --------- Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
1 parent 7833a79 commit 08f70b7

58 files changed

Lines changed: 1259 additions & 67 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎x-pack/plugins/integration_assistant/__jest__/fixtures/categorization.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* 2.0.
66
*/
77

8+
import { SamplesFormatName } from '../../common/api/model/common_attributes';
89
import type { Pipeline } from '../../common';
910

1011
export const categorizationInitialPipeline: Pipeline = {
@@ -191,6 +192,7 @@ export const categorizationTestState = {
191192
invalidCategorization: [{ test: 'testinvalid' }],
192193
initialPipeline: categorizationInitialPipeline,
193194
results: { test: 'testresults' },
195+
samplesFormat: { name: SamplesFormatName.Values.json },
194196
};
195197

196198
export const categorizationMockProcessors = [

‎x-pack/plugins/integration_assistant/__jest__/fixtures/ecs_mapping.ts‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
* 2.0.
66
*/
77

8+
import { SamplesFormatName } from '../../common/api/model/common_attributes';
9+
810
export const ecsMappingExpectedResults = {
911
mapping: {
1012
mysql_enterprise: {
@@ -63,21 +65,35 @@ export const ecsMappingExpectedResults = {
6365
value: '8.11.0',
6466
},
6567
},
68+
{
69+
set: {
70+
copy_from: 'message',
71+
field: 'originalMessage',
72+
tag: 'copy_original_message',
73+
},
74+
},
6675
{
6776
rename: {
68-
field: 'message',
77+
field: 'originalMessage',
6978
target_field: 'event.original',
7079
tag: 'rename_message',
7180
ignore_missing: true,
7281
if: 'ctx.event?.original == null',
7382
},
7483
},
84+
{
85+
remove: {
86+
field: 'originalMessage',
87+
if: 'ctx.event?.original != null',
88+
ignore_missing: true,
89+
tag: 'remove_copied_message',
90+
},
91+
},
7592
{
7693
remove: {
7794
field: 'message',
7895
ignore_missing: true,
7996
tag: 'remove_message',
80-
if: 'ctx.event?.original != null',
8197
},
8298
},
8399
{
@@ -450,7 +466,7 @@ export const ecsTestState = {
450466
finalMapping: { test: 'testmapping' },
451467
sampleChunks: [''],
452468
results: { test: 'testresults' },
453-
samplesFormat: 'testsamplesFormat',
469+
samplesFormat: { name: SamplesFormatName.Values.json },
454470
ecsVersion: 'testversion',
455471
chunkMapping: { test1: 'test1' },
456472
useFinalMapping: false,
@@ -462,4 +478,5 @@ export const ecsTestState = {
462478
packageName: 'testpackage',
463479
dataStreamName: 'testDataStream',
464480
combinedSamples: '{"test1": "test1"}',
481+
additionalProcessors: [],
465482
};

‎x-pack/plugins/integration_assistant/__jest__/fixtures/kv.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { SamplesFormatName } from '../../common/api/model/common_attributes';
9+
10+
export const kvState = {
11+
lastExecutedChain: 'testchain',
12+
packageName: 'testPackage',
13+
dataStreamName: 'testDatastream',
14+
kvProcessor: { kv: { field: 'test', target_field: 'newtest' } },
15+
logSamples: ['<134>1 dummy="data"'],
16+
jsonSamples: ['{"test1": "test1"}'],
17+
kvLogMessages: ['{"test1": "test1"}'],
18+
finalized: false,
19+
samplesFormat: { name: SamplesFormatName.Values.structured },
20+
header: true,
21+
ecsVersion: 'testVersion',
22+
errors: { test: 'testerror' },
23+
additionalProcessors: [{ kv: { field: 'test', target_field: 'newtest' } }],
24+
};

‎x-pack/plugins/integration_assistant/__jest__/fixtures/log_type_detection.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,14 @@ import { SamplesFormatName } from '../../common/api/model/common_attributes';
1010
export const logFormatDetectionTestState = {
1111
lastExecutedChain: 'testchain',
1212
logSamples: ['{"test1": "test1"}'],
13+
jsonSamples: ['{"test1": "test1"}'],
1314
exAnswer: 'testanswer',
1415
packageName: 'testPackage',
1516
dataStreamName: 'testDatastream',
1617
finalized: false,
17-
samplesFormat: { name: SamplesFormatName.Values.json },
18+
samplesFormat: { name: SamplesFormatName.Values.structured },
19+
header: true,
1820
ecsVersion: 'testVersion',
1921
results: { test1: 'test1' },
22+
additionalProcessors: [{ kv: { field: 'test', target_field: 'newtest' } }],
2023
};

‎x-pack/plugins/integration_assistant/__jest__/fixtures/related.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
* 2.0.
66
*/
77

8+
import { SamplesFormatName } from '../../common/api/model/common_attributes';
89
import type { Pipeline } from '../../common';
910

1011
export const relatedInitialPipeline: Pipeline = {
@@ -166,6 +167,7 @@ export const relatedTestState = {
166167
initialPipeline: relatedInitialPipeline,
167168
results: { test: 'testresults' },
168169
lastExecutedChain: 'testchain',
170+
samplesFormat: { name: SamplesFormatName.Values.json },
169171
};
170172

171173
export const relatedMockProcessors = [

‎x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.schema.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,13 @@ paths:
2020
required:
2121
- logSamples
2222
- connectorId
23+
- packageName
24+
- dataStreamName
2325
properties:
26+
packageName:
27+
$ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
28+
dataStreamName:
29+
$ref: "../model/common_attributes.schema.yaml#/components/schemas/DataStreamName"
2430
logSamples:
2531
$ref: "../model/common_attributes.schema.yaml#/components/schemas/LogSamples"
2632
connectorId:

‎x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { expectParseSuccess } from '@kbn/zod-helpers';
9+
import { AnalyzeLogsRequestBody } from './analyze_logs_route';
10+
import { getAnalyzeLogsRequestBody } from '../model/api_test.mock';
11+
12+
describe('Analyze Logs request schema', () => {
13+
test('full request validate', () => {
14+
const payload: AnalyzeLogsRequestBody = getAnalyzeLogsRequestBody();
15+
16+
const result = AnalyzeLogsRequestBody.safeParse(payload);
17+
expectParseSuccess(result);
18+
expect(result.data).toEqual(payload);
19+
});
20+
});

‎x-pack/plugins/integration_assistant/common/api/analyze_logs/analyze_logs_route.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,19 @@
1616

1717
import { z } from '@kbn/zod';
1818

19-
import { LogSamples, Connector, LangSmithOptions } from '../model/common_attributes';
19+
import {
20+
LogSamples,
21+
Connector,
22+
LangSmithOptions,
23+
DataStreamName,
24+
PackageName,
25+
} from '../model/common_attributes';
2026
import { AnalyzeLogsAPIResponse } from '../model/response_schemas';
2127

2228
export type AnalyzeLogsRequestBody = z.infer<typeof AnalyzeLogsRequestBody>;
2329
export const AnalyzeLogsRequestBody = z.object({
30+
packageName: PackageName,
31+
dataStreamName: DataStreamName,
2432
logSamples: LogSamples,
2533
connectorId: Connector,
2634
langSmithOptions: LangSmithOptions.optional(),

‎x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.schema.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ paths:
2323
- rawSamples
2424
- currentPipeline
2525
- connectorId
26+
- samplesFormat
2627
properties:
2728
packageName:
2829
$ref: "../model/common_attributes.schema.yaml#/components/schemas/PackageName"
@@ -34,6 +35,8 @@ paths:
3435
$ref: "../model/common_attributes.schema.yaml#/components/schemas/Pipeline"
3536
connectorId:
3637
$ref: "../model/common_attributes.schema.yaml#/components/schemas/Connector"
38+
samplesFormat:
39+
$ref: "../model/common_attributes.schema.yaml#/components/schemas/SamplesFormat"
3740
langSmithOptions:
3841
$ref: "../model/common_attributes.schema.yaml#/components/schemas/LangSmithOptions"
3942
responses:

‎x-pack/plugins/integration_assistant/common/api/categorization/categorization_route.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
PackageName,
1515
Pipeline,
1616
RawSamples,
17+
SamplesFormat,
1718
} from '../model/common_attributes';
1819
import { CategorizationAPIResponse } from '../model/response_schemas';
1920

@@ -22,6 +23,7 @@ export const CategorizationRequestBody = z.object({
2223
packageName: PackageName,
2324
dataStreamName: DataStreamName,
2425
rawSamples: RawSamples,
26+
samplesFormat: SamplesFormat,
2527
currentPipeline: Pipeline,
2628
connectorId: Connector,
2729
langSmithOptions: LangSmithOptions.optional(),

0 commit comments

Comments
 (0)