Skip to content

Commit 45558c3

Browse files
uri-weismanuri-weisman
authored
[Entity Store] Populate entity.source from integration's domain (#259813)
1 parent 584577b commit 45558c3

24 files changed

Lines changed: 388 additions & 134 deletions

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/common_fields.ts‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,16 @@
66
*/
77

88
import type { Condition } from '@kbn/streamlang';
9-
import type { EntityType, EntityField } from './entity_schema';
9+
import type { EntityType, EntityField, FieldEvaluation } from './entity_schema';
1010
import { collectValues, newestValue, oldestValue } from './field_retention_operations';
1111

1212
export const ENTITY_ID_FIELD = 'entity.id';
13+
export const ENTITY_SOURCE_FIELD = 'entity.source';
1314
// Copied from x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_definitions/entity_descriptions/common.ts
1415

1516
export const getCommonFieldDescriptions = (
1617
ecsField: Omit<EntityType, 'generic'> | 'entity'
1718
): EntityField[] => [
18-
newestValue({
19-
source: '_index',
20-
destination: 'entity.source',
21-
}),
2219
newestValue({ source: 'asset.id' }),
2320
newestValue({ source: 'asset.name' }),
2421
newestValue({ source: 'asset.owner' }),
@@ -52,7 +49,10 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => {
5249
const prefix = rootField ? `${rootField}.entity` : 'entity';
5350

5451
return [
55-
newestValue({ source: `${prefix}.source`, destination: 'entity.source' }),
52+
collectValues({ source: 'event.module' }),
53+
collectValues({ source: 'event.dataset' }),
54+
collectValues({ source: 'data_stream.dataset', fieldHistoryLength: 50 }),
55+
collectValues({ source: ENTITY_SOURCE_FIELD, fieldHistoryLength: 50 }),
5656
newestValue({ source: `${prefix}.type`, destination: 'entity.type' }),
5757
newestValue({ source: `${prefix}.sub_type`, destination: 'entity.sub_type' }),
5858
newestValue({ source: `${prefix}.url`, destination: 'entity.url' }),
@@ -189,6 +189,17 @@ export const getEntityFieldsDescriptions = (rootField?: EntityType) => {
189189
];
190190
};
191191

192+
export const ENTITY_SOURCE_FIELD_EVALUATION: FieldEvaluation = {
193+
destination: ENTITY_SOURCE_FIELD,
194+
sources: [
195+
{ field: 'event.module' },
196+
{ field: 'event.dataset' },
197+
{ field: 'data_stream.dataset' },
198+
],
199+
fallbackValue: null,
200+
whenClauses: [],
201+
};
202+
192203
export function isNotEmptyCondition(field: string): Condition {
193204
return {
194205
and: [

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.gen.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export const EntityField = z
3535
name: z.string().optional(),
3636
type: z.string().optional(),
3737
sub_type: z.string().optional(),
38-
source: z.string().optional(),
38+
source: z.array(z.string()).optional(),
3939
EngineMetadata: EngineMetadata.optional(),
4040
attributes: z
4141
.object({

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity.schema.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ components:
3030
sub_type:
3131
type: string
3232
source:
33-
type: string
33+
type: array
34+
items:
35+
type: string
3436
EngineMetadata:
3537
$ref: '#/components/schemas/EngineMetadata'
3638
attributes:

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/entity_schema.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ const fieldEvaluationSourceSchema = z.union([
5151
const fieldEvaluationSchema = z.object({
5252
destination: z.string(),
5353
sources: z.array(fieldEvaluationSourceSchema),
54-
fallbackValue: z.string(),
54+
fallbackValue: z.string().nullable(),
5555
whenClauses: z.array(fieldEvaluationWhenClauseSchema),
5656
});
5757

@@ -136,6 +136,8 @@ export const entitySchema = z.object({
136136
filter: z.string().optional(),
137137
entityTypeFallback: z.string().optional(),
138138
fields: z.array(fieldSchema),
139+
// Optional evaluated fields applied before pre-agg overrides and STATS for all entity types.
140+
fieldEvaluations: z.optional(z.array(fieldEvaluationSchema)),
139141
identityField: identityFieldSchema,
140142
indexPatterns: z.array(z.string()),
141143
// Optional filter (Condition from @kbn/streamlang) applied in ESQL only, right after the

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/generic.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,18 @@
77

88
import { newestValue } from './field_retention_operations';
99
import type { EntityDefinitionWithoutId } from './entity_schema';
10-
import { getCommonFieldDescriptions, getEntityFieldsDescriptions } from './common_fields';
10+
import {
11+
ENTITY_SOURCE_FIELD_EVALUATION,
12+
getCommonFieldDescriptions,
13+
getEntityFieldsDescriptions,
14+
} from './common_fields';
1115

12-
export const genericEntityDefinition: EntityDefinitionWithoutId = {
16+
export const genericEntityDefinition = {
1317
type: 'generic',
1418
name: `Security 'generic' Entity Store Definition`,
1519
identityField: { singleField: 'entity.id', skipTypePrepend: true },
1620
indexPatterns: [],
21+
fieldEvaluations: [ENTITY_SOURCE_FIELD_EVALUATION],
1722
fields: [
1823
// We want this to make sure it's also extracted on CCS logs extraction
1924
newestValue({ source: 'entity.id' }),
@@ -50,4 +55,4 @@ export const genericEntityDefinition: EntityDefinitionWithoutId = {
5055

5156
...getCommonFieldDescriptions('entity'),
5257
],
53-
} as const satisfies EntityDefinitionWithoutId;
58+
} satisfies EntityDefinitionWithoutId;

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/host.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import { collectValues as collect, newestValue, oldestValue } from './field_retention_operations';
99
import type { EntityDefinitionWithoutId } from './entity_schema';
1010
import {
11+
ENTITY_SOURCE_FIELD_EVALUATION,
1112
getCommonFieldDescriptions,
1213
getEntityFieldsDescriptions,
1314
isNotEmptyCondition,
@@ -36,6 +37,7 @@ export const hostEntityDefinition: EntityDefinitionWithoutId = {
3637
},
3738
entityTypeFallback: 'Host',
3839
indexPatterns: [],
40+
fieldEvaluations: [ENTITY_SOURCE_FIELD_EVALUATION],
3941
fields: [
4042
newestValue({ destination: 'entity.name', source: 'host.name' }),
4143
oldestValue({ source: 'host.entity.id' }),

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/service.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@
55
* 2.0.
66
*/
77

8-
import { getCommonFieldDescriptions, getEntityFieldsDescriptions } from './common_fields';
8+
import {
9+
ENTITY_SOURCE_FIELD_EVALUATION,
10+
getCommonFieldDescriptions,
11+
getEntityFieldsDescriptions,
12+
} from './common_fields';
913
import type { EntityDefinitionWithoutId } from './entity_schema';
1014
import { collectValues as collect, newestValue, oldestValue } from './field_retention_operations';
1115

@@ -15,6 +19,7 @@ export const serviceEntityDefinition: EntityDefinitionWithoutId = {
1519
identityField: { singleField: 'service.name' },
1620
indexPatterns: [],
1721
entityTypeFallback: 'Service',
22+
fieldEvaluations: [ENTITY_SOURCE_FIELD_EVALUATION],
1823
fields: [
1924
newestValue({ destination: 'entity.name', source: 'service.name' }),
2025
oldestValue({ source: 'service.entity.id' }),

‎x-pack/solutions/security/plugins/entity_store/common/domain/definitions/user.ts‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import type { Condition } from '@kbn/streamlang';
99
import {
10+
ENTITY_SOURCE_FIELD_EVALUATION,
1011
fieldNotOneOfCondition,
1112
getCommonFieldDescriptions,
1213
getEntityFieldsDescriptions,
@@ -75,6 +76,7 @@ const nonIdpPostAggFilter = nonIdpDocumentFilter;
7576
export const userEntityDefinition: EntityDefinitionWithoutId = {
7677
type: 'user',
7778
name: `Security 'user' Entity Store Definition`,
79+
fieldEvaluations: [ENTITY_SOURCE_FIELD_EVALUATION],
7880
identityField: {
7981
fieldEvaluations: [
8082
{
@@ -186,14 +188,6 @@ export const userEntityDefinition: EntityDefinitionWithoutId = {
186188
],
187189
fields: [
188190
newestValue({ source: 'entity.name' }),
189-
// Having multiple values in event.module or data_stream.dataset is a good feature
190-
// but causes complexity for CCS extraction.
191-
// That's why event.module and data_stream.dataset always use MV_FIRST on its usage
192-
collect({ source: 'event.module' }),
193-
// keep field length large for safety to not lose idps
194-
// with many datasets
195-
collect({ source: 'data_stream.dataset', fieldHistoryLength: 50 }),
196-
197191
collect({ source: 'event.kind' }),
198192
collect({ source: 'event.category' }),
199193
collect({ source: 'event.type' }),

‎x-pack/solutions/security/plugins/entity_store/common/domain/euid/commons.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import type {
1111
CalculatedEntityIdentity,
1212
EntityDefinitionWithoutId,
1313
EuidAttribute,
14+
EuidField,
15+
EuidSeparator,
1416
FieldEvaluationSource,
1517
FieldValueSchema,
1618
} from '../definitions/entity_schema';
@@ -234,11 +236,11 @@ export function getFieldsToBeFilteredOut(
234236
return toFilterOut;
235237
}
236238

237-
export function isEuidField(attr: EuidAttribute) {
239+
export function isEuidField(attr: EuidAttribute): attr is EuidField {
238240
return 'field' in attr;
239241
}
240242

241-
export function isEuidSeparator(attr: EuidAttribute) {
243+
export function isEuidSeparator(attr: EuidAttribute): attr is EuidSeparator {
242244
return 'sep' in attr;
243245
}
244246

‎x-pack/solutions/security/plugins/entity_store/common/domain/euid/dsl.ts‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
import type { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
99
import { conditionToQueryDsl } from '@kbn/streamlang';
10-
import type { EntityType } from '../definitions/entity_schema';
10+
import type { EntityType, FieldEvaluation } from '../definitions/entity_schema';
1111
import { isSingleFieldIdentity } from '../definitions/entity_schema';
1212
import { getEntityDefinitionWithoutId } from '../definitions/registry';
1313
import { isNotEmptyCondition } from '../definitions/common_fields';
@@ -27,7 +27,6 @@ import {
2727
getSourceMatchSpec,
2828
type SourceMatchSpec,
2929
} from './field_evaluations';
30-
import type { FieldEvaluation } from '../definitions/entity_schema';
3130

3231
/**
3332
* Returns a DSL filter that matches documents considered for the given entity type.
@@ -118,8 +117,9 @@ export function getEuidDslFilterBasedOnDocument(
118117
};
119118
}
120119

121-
if (identityField.fieldEvaluations?.length) {
122-
const evaluated = applyFieldEvaluations(doc, identityField.fieldEvaluations);
120+
const fieldEvaluations = identityField.fieldEvaluations ?? [];
121+
if (fieldEvaluations.length > 0) {
122+
const evaluated = applyFieldEvaluations(doc, fieldEvaluations);
123123
doc = { ...doc, ...evaluated };
124124
}
125125
if (entityDefinition.whenConditionTrueSetFieldsPreAgg?.length) {
@@ -139,9 +139,7 @@ export function getEuidDslFilterBasedOnDocument(
139139

140140
// Evaluated fields (e.g. entity.namespace from event.module) are computed in memory and are not
141141
// stored in the index. Including them in the query would make it never match real documents.
142-
const evaluatedDestinations = new Set(
143-
identityField.fieldEvaluations?.map((e) => e.destination) ?? []
144-
);
142+
const evaluatedDestinations = new Set(fieldEvaluations.map((e) => e.destination));
145143

146144
const filterValues = Object.entries(fieldsToBeFilteredOn.values).filter(
147145
([field]) => !evaluatedDestinations.has(field)
@@ -153,21 +151,23 @@ export function getEuidDslFilterBasedOnDocument(
153151
})),
154152
},
155153
};
154+
const boolQuery = dsl.bool!;
156155

157156
const toBeFilteredOut = getFieldsToBeFilteredOut(effectiveRanking, fieldsToBeFilteredOn).filter(
158157
(field) => !evaluatedDestinations.has(field)
159158
);
160159
if (toBeFilteredOut.length > 0) {
161-
const priorMust = Array.isArray(dsl.bool?.must) ? dsl.bool.must : [];
160+
const priorMust = Array.isArray(boolQuery.must) ? boolQuery.must : [];
162161
dsl.bool = {
163-
...dsl.bool,
162+
...boolQuery,
164163
must: [...priorMust, ...toBeFilteredOut.map(fieldMissingOrEmptyDsl)],
165164
};
166165
}
167166

168-
if (identityField.fieldEvaluations?.length) {
169-
const filterList = Array.isArray(dsl.bool?.filter) ? dsl.bool.filter : [];
170-
for (const evaluation of identityField.fieldEvaluations) {
167+
if (fieldEvaluations.length > 0) {
168+
const currentBoolQuery = dsl.bool!;
169+
const filterList = Array.isArray(currentBoolQuery.filter) ? currentBoolQuery.filter : [];
170+
for (const evaluation of fieldEvaluations) {
171171
const { exactMatchFields, prefixMatchFields } = getSourceFieldNames(evaluation.sources);
172172
const sourceFields = [...exactMatchFields, ...prefixMatchFields];
173173
const hasEvaluatedSource = sourceFields.some((f) => evaluatedDestinations.has(f));

0 commit comments

Comments
 (0)