Skip to content

Commit 80d0eb6

Browse files
kibanamachineP1llus
kibanamachine
and
P1llus
authored
[9.0] [Rule Migration] Add telemetry events to translation graphs (#209352) (#209915)
# Backport This will backport the following commits from `main` to `9.0`: - [[Rule Migration] Add telemetry events to translation graphs (#209352)](#209352) <!--- Backport version: 9.4.3 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Marius Iversen","email":"marius.iversen@elastic.co"},"sourceCommit":{"committedDate":"2025-02-05T21:12:50Z","message":"[Rule Migration] Add telemetry events to translation graphs (#209352)\n\n## Summary\n\nThis PR adds telemetry events to SIEM migration backend using the event\nbased telemetry already existing in security solutions.\n\nHere is a list of events:\n\n```typescript\nexport const SIEM_MIGRATIONS_MIGRATION_SUCCESS: EventTypeOpts<{\n model: string;\n migrationId: string;\n duration: number;\n completed: number;\n failed: number;\n total: number;\n}\n\nexport const SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS: EventTypeOpts<{\n model: string;\n migrationId: string;\n duration: number;\n translationResult: string;\n prebuiltMatch: boolean;\n}\n\nexport const SIEM_MIGRATIONS_PREBUILT_RULES_MATCH: EventTypeOpts<{\n model: string;\n migrationId: string;\n preFilterRuleNames: string[];\n preFilterRuleCount: number;\n postFilterRuleName: string;\n postFilterRuleCount: number;\n}\n\nexport const SIEM_MIGRATIONS_INTEGRATIONS_MATCH: EventTypeOpts<{\n model: string;\n migrationId: string;\n preFilterIntegrationNames: string[];\n preFilterIntegrationCount: number;\n postFilterIntegrationName: string;\n postFilterIntegrationCount: number;\n}\n\nexport const SIEM_MIGRATIONS_MIGRATION_FAILURE: EventTypeOpts<{\n model: string;\n error: string;\n migrationId: string;\n duration: number;\n completed: number;\n failed: number;\n total: number;\n}\n\nexport const SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE: EventTypeOpts<{\n model: string;\n error: string;\n migrationId: string;\n}\n```","sha":"6cab1dc6f8cd1f48b17a1ad7546c65cbbd79f4a8","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team: SecuritySolution","backport:version","v8.18.0","v9.1.0","v8.19.0"],"title":"[Rule Migration] Add telemetry events to translation graphs","number":209352,"url":"https://github.com/elastic/kibana/pull/209352","mergeCommit":{"message":"[Rule Migration] Add telemetry events to translation graphs (#209352)\n\n## Summary\n\nThis PR adds telemetry events to SIEM migration backend using the event\nbased telemetry already existing in security solutions.\n\nHere is a list of events:\n\n```typescript\nexport const SIEM_MIGRATIONS_MIGRATION_SUCCESS: EventTypeOpts<{\n model: string;\n migrationId: string;\n duration: number;\n completed: number;\n failed: number;\n total: number;\n}\n\nexport const SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS: EventTypeOpts<{\n model: string;\n migrationId: string;\n duration: number;\n translationResult: string;\n prebuiltMatch: boolean;\n}\n\nexport const SIEM_MIGRATIONS_PREBUILT_RULES_MATCH: EventTypeOpts<{\n model: string;\n migrationId: string;\n preFilterRuleNames: string[];\n preFilterRuleCount: number;\n postFilterRuleName: string;\n postFilterRuleCount: number;\n}\n\nexport const SIEM_MIGRATIONS_INTEGRATIONS_MATCH: EventTypeOpts<{\n model: string;\n migrationId: string;\n preFilterIntegrationNames: string[];\n preFilterIntegrationCount: number;\n postFilterIntegrationName: string;\n postFilterIntegrationCount: number;\n}\n\nexport const SIEM_MIGRATIONS_MIGRATION_FAILURE: EventTypeOpts<{\n model: string;\n error: string;\n migrationId: string;\n duration: number;\n completed: number;\n failed: number;\n total: number;\n}\n\nexport const SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE: EventTypeOpts<{\n model: string;\n error: string;\n migrationId: string;\n}\n```","sha":"6cab1dc6f8cd1f48b17a1ad7546c65cbbd79f4a8"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.18","8.x"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/209352","number":209352,"mergeCommit":{"message":"[Rule Migration] Add telemetry events to translation graphs (#209352)\n\n## Summary\n\nThis PR adds telemetry events to SIEM migration backend using the event\nbased telemetry already existing in security solutions.\n\nHere is a list of events:\n\n```typescript\nexport const SIEM_MIGRATIONS_MIGRATION_SUCCESS: EventTypeOpts<{\n model: string;\n migrationId: string;\n duration: number;\n completed: number;\n failed: number;\n total: number;\n}\n\nexport const SIEM_MIGRATIONS_RULE_TRANSLATION_SUCCESS: EventTypeOpts<{\n model: string;\n migrationId: string;\n duration: number;\n translationResult: string;\n prebuiltMatch: boolean;\n}\n\nexport const SIEM_MIGRATIONS_PREBUILT_RULES_MATCH: EventTypeOpts<{\n model: string;\n migrationId: string;\n preFilterRuleNames: string[];\n preFilterRuleCount: number;\n postFilterRuleName: string;\n postFilterRuleCount: number;\n}\n\nexport const SIEM_MIGRATIONS_INTEGRATIONS_MATCH: EventTypeOpts<{\n model: string;\n migrationId: string;\n preFilterIntegrationNames: string[];\n preFilterIntegrationCount: number;\n postFilterIntegrationName: string;\n postFilterIntegrationCount: number;\n}\n\nexport const SIEM_MIGRATIONS_MIGRATION_FAILURE: EventTypeOpts<{\n model: string;\n error: string;\n migrationId: string;\n duration: number;\n completed: number;\n failed: number;\n total: number;\n}\n\nexport const SIEM_MIGRATIONS_RULE_TRANSLATION_FAILURE: EventTypeOpts<{\n model: string;\n error: string;\n migrationId: string;\n}\n```","sha":"6cab1dc6f8cd1f48b17a1ad7546c65cbbd79f4a8"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> Co-authored-by: Marius Iversen <marius.iversen@elastic.co>
1 parent 67a7a2f commit 80d0eb6

18 files changed

Lines changed: 571 additions & 79 deletions

File tree

‎x-pack/solutions/security/plugins/security_solution/docs/siem_migration/img/agent_graph.png‎

3.14 KB
Loading

‎x-pack/solutions/security/plugins/security_solution/scripts/siem_migration/draw_graphs_script.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import fs from 'fs/promises';
1717
import path from 'path';
1818
import { getRuleMigrationAgent } from '../../server/lib/siem_migrations/rules/task/agent';
1919
import type { RuleMigrationsRetriever } from '../../server/lib/siem_migrations/rules/task/retrievers';
20+
import type { SiemMigrationTelemetryClient } from '../../server/lib/siem_migrations/rules/task/rule_migrations_telemetry_client';
2021

2122
interface Drawable {
2223
drawMermaidPng: () => Promise<Blob>;
@@ -36,12 +37,14 @@ const createLlmInstance = () => {
3637

3738
async function getAgentGraph(logger: Logger): Promise<Drawable> {
3839
const model = createLlmInstance();
40+
const telemetryClient = {} as SiemMigrationTelemetryClient;
3941
const graph = getRuleMigrationAgent({
4042
model,
4143
inferenceClient,
4244
ruleMigrationsRetriever,
4345
connectorId,
4446
logger,
47+
telemetryClient,
4548
});
4649
return graph.getGraphAsync({ xray: true });
4750
}

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,14 @@ import type {
1313
import { loggerMock } from '@kbn/logging-mocks';
1414
import { FakeLLM } from '@langchain/core/utils/testing';
1515
import type { RuleMigrationsRetriever } from '../retrievers';
16+
import type { SiemMigrationTelemetryClient } from '../rule_migrations_telemetry_client';
1617
import { getRuleMigrationAgent } from './graph';
1718

1819
describe('getRuleMigrationAgent', () => {
1920
const model = new FakeLLM({
2021
response: JSON.stringify({}, null, 2),
2122
}) as unknown as ActionsClientChatOpenAI | ActionsClientSimpleChatModel;
22-
23+
const telemetryClient = {} as SiemMigrationTelemetryClient;
2324
const inferenceClient = {} as InferenceClient;
2425
const connectorId = 'draw_graphs';
2526
const ruleMigrationsRetriever = {} as RuleMigrationsRetriever;
@@ -33,6 +34,7 @@ describe('getRuleMigrationAgent', () => {
3334
ruleMigrationsRetriever,
3435
connectorId,
3536
logger,
37+
telemetryClient,
3638
});
3739
} catch (error) {
3840
throw Error(`getRuleMigrationAgent threw an error: ${error}`);

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/graph.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,28 +8,29 @@
88
import { END, START, StateGraph } from '@langchain/langgraph';
99
import { getCreateSemanticQueryNode } from './nodes/create_semantic_query';
1010
import { getMatchPrebuiltRuleNode } from './nodes/match_prebuilt_rule';
11-
1211
import { migrateRuleState } from './state';
1312
import { getTranslateRuleGraph } from './sub_graphs/translate_rule';
1413
import type { MigrateRuleGraphParams, MigrateRuleState } from './types';
15-
1614
export function getRuleMigrationAgent({
1715
model,
1816
inferenceClient,
1917
ruleMigrationsRetriever,
2018
connectorId,
2119
logger,
20+
telemetryClient,
2221
}: MigrateRuleGraphParams) {
2322
const matchPrebuiltRuleNode = getMatchPrebuiltRuleNode({
2423
model,
2524
logger,
2625
ruleMigrationsRetriever,
26+
telemetryClient,
2727
});
2828
const translationSubGraph = getTranslateRuleGraph({
2929
model,
3030
inferenceClient,
3131
ruleMigrationsRetriever,
3232
connectorId,
33+
telemetryClient,
3334
logger,
3435
});
3536
const createSemanticQueryNode = getCreateSemanticQueryNode({ model });

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/nodes/match_prebuilt_rule/match_prebuilt_rule.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,16 @@ import {
1313
RuleTranslationResult,
1414
} from '../../../../../../../../common/siem_migrations/constants';
1515
import type { RuleMigrationsRetriever } from '../../../retrievers';
16+
import type { SiemMigrationTelemetryClient } from '../../../rule_migrations_telemetry_client';
1617
import type { ChatModel } from '../../../util/actions_client_chat';
18+
import { cleanMarkdown, generateAssistantComment } from '../../../util/comments';
1719
import type { GraphNode } from '../../types';
1820
import { MATCH_PREBUILT_RULE_PROMPT } from './prompts';
19-
import { cleanMarkdown, generateAssistantComment } from '../../../util/comments';
2021

2122
interface GetMatchPrebuiltRuleNodeParams {
2223
model: ChatModel;
2324
logger: Logger;
25+
telemetryClient: SiemMigrationTelemetryClient;
2426
ruleMigrationsRetriever: RuleMigrationsRetriever;
2527
}
2628

@@ -32,6 +34,7 @@ interface GetMatchedRuleResponse {
3234
export const getMatchPrebuiltRuleNode = ({
3335
model,
3436
ruleMigrationsRetriever,
37+
telemetryClient,
3538
logger,
3639
}: GetMatchPrebuiltRuleNodeParams): GraphNode => {
3740
return async (state) => {
@@ -42,6 +45,8 @@ export const getMatchPrebuiltRuleNode = ({
4245
techniqueIds.join(',')
4346
);
4447
if (prebuiltRules.length === 0) {
48+
telemetryClient.reportPrebuiltRulesMatch({ preFilterRules: [] });
49+
4550
return {
4651
comments: [
4752
generateAssistantComment(
@@ -80,6 +85,10 @@ export const getMatchPrebuiltRuleNode = ({
8085

8186
if (response.match) {
8287
const matchedRule = prebuiltRules.find((r) => r.name === response.match);
88+
telemetryClient.reportPrebuiltRulesMatch({
89+
preFilterRules: prebuiltRules,
90+
postFilterRule: matchedRule,
91+
});
8392
if (matchedRule) {
8493
return {
8594
comments,

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/graph.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ import { END, START, StateGraph } from '@langchain/langgraph';
99
import { isEmpty } from 'lodash/fp';
1010
import { RuleTranslationResult } from '../../../../../../../../common/siem_migrations/constants';
1111
import { getEcsMappingNode } from './nodes/ecs_mapping';
12-
import { translationResultNode } from './nodes/translation_result';
1312
import { getFixQueryErrorsNode } from './nodes/fix_query_errors';
13+
import { getInlineQueryNode } from './nodes/inline_query';
1414
import { getRetrieveIntegrationsNode } from './nodes/retrieve_integrations';
1515
import { getTranslateRuleNode } from './nodes/translate_rule';
16+
import { getTranslationResultNode } from './nodes/translation_result';
1617
import { getValidationNode } from './nodes/validation';
17-
import { getInlineQueryNode } from './nodes/inline_query';
1818
import { translateRuleState } from './state';
1919
import type { TranslateRuleGraphParams, TranslateRuleState } from './types';
2020

@@ -27,16 +27,22 @@ export function getTranslateRuleGraph({
2727
connectorId,
2828
ruleMigrationsRetriever,
2929
logger,
30+
telemetryClient,
3031
}: TranslateRuleGraphParams) {
3132
const translateRuleNode = getTranslateRuleNode({
3233
inferenceClient,
3334
connectorId,
3435
logger,
3536
});
37+
const translationResultNode = getTranslationResultNode();
3638
const inlineQueryNode = getInlineQueryNode({ model, ruleMigrationsRetriever });
3739
const validationNode = getValidationNode({ logger });
3840
const fixQueryErrorsNode = getFixQueryErrorsNode({ inferenceClient, connectorId, logger });
39-
const retrieveIntegrationsNode = getRetrieveIntegrationsNode({ model, ruleMigrationsRetriever });
41+
const retrieveIntegrationsNode = getRetrieveIntegrationsNode({
42+
model,
43+
ruleMigrationsRetriever,
44+
telemetryClient,
45+
});
4046
const ecsMappingNode = getEcsMappingNode({ inferenceClient, connectorId, logger });
4147

4248
const translateRuleGraph = new StateGraph(translateRuleState)

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/retrieve_integrations/retrieve_integrations.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,15 @@
77

88
import { JsonOutputParser } from '@langchain/core/output_parsers';
99
import type { RuleMigrationsRetriever } from '../../../../../retrievers';
10+
import type { SiemMigrationTelemetryClient } from '../../../../../rule_migrations_telemetry_client';
1011
import type { ChatModel } from '../../../../../util/actions_client_chat';
12+
import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
1113
import type { GraphNode } from '../../types';
1214
import { MATCH_INTEGRATION_PROMPT } from './prompts';
13-
import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
1415

1516
interface GetRetrieveIntegrationsNodeParams {
1617
model: ChatModel;
18+
telemetryClient: SiemMigrationTelemetryClient;
1719
ruleMigrationsRetriever: RuleMigrationsRetriever;
1820
}
1921

@@ -25,12 +27,16 @@ interface GetMatchedIntegrationResponse {
2527
export const getRetrieveIntegrationsNode = ({
2628
model,
2729
ruleMigrationsRetriever,
30+
telemetryClient,
2831
}: GetRetrieveIntegrationsNodeParams): GraphNode => {
2932
return async (state) => {
3033
const query = state.semantic_query;
3134

3235
const integrations = await ruleMigrationsRetriever.integrations.getIntegrations(query);
3336
if (integrations.length === 0) {
37+
telemetryClient.reportIntegrationsMatch({
38+
preFilterIntegrations: [],
39+
});
3440
return {
3541
comments: [
3642
generateAssistantComment(
@@ -67,6 +73,10 @@ export const getRetrieveIntegrationsNode = ({
6773

6874
if (response.match) {
6975
const matchedIntegration = integrations.find((r) => r.title === response.match);
76+
telemetryClient.reportIntegrationsMatch({
77+
preFilterIntegrations: integrations,
78+
postFilterIntegration: matchedIntegration,
79+
});
7080
if (matchedIntegration) {
7181
return { integration: matchedIntegration, comments };
7282
}

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translate_rule/translate_rule.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77

88
import type { Logger } from '@kbn/core/server';
99
import type { InferenceClient } from '@kbn/inference-plugin/server';
10+
import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
1011
import { getEsqlKnowledgeBase } from '../../../../../util/esql_knowledge_base_caller';
1112
import type { GraphNode } from '../../types';
1213
import { ESQL_SYNTAX_TRANSLATION_PROMPT } from './prompts';
13-
import { cleanMarkdown, generateAssistantComment } from '../../../../../util/comments';
1414

1515
interface GetTranslateRuleNodeParams {
1616
inferenceClient: InferenceClient;

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44
* 2.0; you may not use this file except in compliance with the Elastic License
55
* 2.0.
66
*/
7-
export { translationResultNode } from './translation_result';
7+
export { getTranslationResultNode } from './translation_result';

‎x-pack/solutions/security/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/translation_result/translation_result.ts‎

Lines changed: 28 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,36 +12,38 @@ import {
1212
} from '../../../../../../../../../../common/siem_migrations/constants';
1313
import type { GraphNode } from '../../types';
1414

15-
export const translationResultNode: GraphNode = async (state) => {
16-
// Set defaults
17-
const elasticRule = {
18-
title: state.original_rule.title,
19-
description: state.original_rule.description || state.original_rule.title,
20-
severity: DEFAULT_TRANSLATION_SEVERITY,
21-
risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
22-
...state.elastic_rule,
23-
};
15+
export const getTranslationResultNode = (): GraphNode => {
16+
return async (state) => {
17+
// Set defaults
18+
const elasticRule = {
19+
title: state.original_rule.title,
20+
description: state.original_rule.description || state.original_rule.title,
21+
severity: DEFAULT_TRANSLATION_SEVERITY,
22+
risk_score: DEFAULT_TRANSLATION_RISK_SCORE,
23+
...state.elastic_rule,
24+
};
2425

25-
const query = elasticRule.query;
26-
let translationResult;
26+
const query = elasticRule.query;
27+
let translationResult;
2728

28-
if (!query) {
29-
translationResult = RuleTranslationResult.UNTRANSLATABLE;
30-
} else {
31-
if (query.startsWith('FROM logs-*')) {
32-
elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]');
33-
translationResult = RuleTranslationResult.PARTIAL;
34-
} else if (state.validation_errors?.esql_errors) {
35-
translationResult = RuleTranslationResult.PARTIAL;
36-
} else if (query.match(/\[(macro|lookup):.*?\]/)) {
37-
translationResult = RuleTranslationResult.PARTIAL;
29+
if (!query) {
30+
translationResult = RuleTranslationResult.UNTRANSLATABLE;
3831
} else {
39-
translationResult = RuleTranslationResult.FULL;
32+
if (query.startsWith('FROM logs-*')) {
33+
elasticRule.query = query.replace('FROM logs-*', 'FROM [indexPattern]');
34+
translationResult = RuleTranslationResult.PARTIAL;
35+
} else if (state.validation_errors?.esql_errors) {
36+
translationResult = RuleTranslationResult.PARTIAL;
37+
} else if (query.match(/\[(macro|lookup):.*?\]/)) {
38+
translationResult = RuleTranslationResult.PARTIAL;
39+
} else {
40+
translationResult = RuleTranslationResult.FULL;
41+
}
4042
}
41-
}
4243

43-
return {
44-
elastic_rule: elasticRule,
45-
translation_result: translationResult,
44+
return {
45+
elastic_rule: elasticRule,
46+
translation_result: translationResult,
47+
};
4648
};
4749
};

0 commit comments

Comments
 (0)