Skip to content

Commit 7fd425f

Browse files
mrodmmrodm
committed
Merge remote-tracking branch 'upstream/main' into test-wolfi-8.16
2 parents 66f9d75 + 4fd64f3 commit 7fd425f

File tree

94 files changed

+6382
-6009
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+6382
-6009
lines changed

‎packages/abnormal_security/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "0.1.2"
3+
changes:
4+
- description: Handles empty threats arrays correctly, ensuring REST calls continue during subsequent intervals.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/10986
27
- version: "0.1.1"
38
changes:
49
- description: Update pagination termination condition in threat data stream.

‎packages/abnormal_security/data_stream/threat/agent/stream/cel.yml.hbs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ redact:
2525
- access_token
2626
program: |
2727
(
28-
has(state.worklist) && size(state.worklist) > 0 ?
28+
has(state.?worklist.threats) && size(state.worklist.threats) > 0 ?
2929
state
3030
:
3131
(

‎packages/abnormal_security/data_stream/threat/sample_event.json‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,22 +45,22 @@
4545
}
4646
},
4747
"agent": {
48-
"ephemeral_id": "b66f399f-ba1c-4fe5-af82-9ca7a0204545",
49-
"id": "e2eadaf0-613d-41d9-913c-96125e06487a",
50-
"name": "elastic-agent-55334",
48+
"ephemeral_id": "900a737b-86e9-4b31-8902-9e933e02c4bc",
49+
"id": "16312af4-ae1e-4ca5-855f-6cb7e433a5a4",
50+
"name": "docker-fleet-agent",
5151
"type": "filebeat",
5252
"version": "8.13.0"
5353
},
5454
"data_stream": {
5555
"dataset": "abnormal_security.threat",
56-
"namespace": "45319",
56+
"namespace": "81591",
5757
"type": "logs"
5858
},
5959
"ecs": {
6060
"version": "8.11.0"
6161
},
6262
"elastic_agent": {
63-
"id": "e2eadaf0-613d-41d9-913c-96125e06487a",
63+
"id": "16312af4-ae1e-4ca5-855f-6cb7e433a5a4",
6464
"snapshot": false,
6565
"version": "8.13.0"
6666
},
@@ -88,7 +88,7 @@
8888
],
8989
"dataset": "abnormal_security.threat",
9090
"id": "2260288475997441000",
91-
"ingested": "2024-08-23T05:40:07Z",
91+
"ingested": "2024-09-03T05:30:00Z",
9292
"kind": "enrichment",
9393
"original": "{\"abxMessageId\":2260288475997441000,\"abxPortalUrl\":\"https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654\",\"attachmentCount\":0,\"attachmentNames\":[],\"attackStrategy\":\"Unknown Sender\",\"attackType\":\"Spam\",\"attackVector\":\"Link\",\"attackedParty\":\"Employee (Other)\",\"autoRemediated\":true,\"ccEmails\":[],\"fromAddress\":\"john@example.com\",\"fromName\":\"john\",\"impersonatedParty\":\"None / Others\",\"internetMessageId\":\"\\u003cAZz8NUMEST-qmuz77_koic@example\\u003e\",\"isRead\":false,\"postRemediated\":false,\"receivedTime\":\"2024-07-17T23:25:38Z\",\"recipientAddress\":\"bob@example.com\",\"remediationStatus\":\"Auto-Remediated\",\"remediationTimestamp\":\"2024-07-17T23:25:45.73564Z\",\"replyToEmails\":[],\"returnPath\":\"bounce-bob_H181S7GUCF@example.com\",\"senderDomain\":\"example.com\",\"senderIpAddress\":\"81.2.69.142\",\"sentTime\":\"2024-07-17T23:25:29Z\",\"subject\":\"YoU.have.𝗪𝟬0𝗡𝗡 a K0baIt 215-piece_ToooI_Set_Noo0wW..#GBOB\",\"summaryInsights\":[\"Abnormal Email Body HTML\",\"Invisible characters found in Email\",\"Suspicious Link\",\"Unusual Sender\",\"Unusual Sender Domain\"],\"threatId\":\"bf255f2d-a2ad-3f50-5075-fdcc24308bbd\",\"toAddresses\":[\"bob@example.com\"],\"urlCount\":1,\"urls\":[\"https://www.example.com/\"]}",
9494
"reference": "https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654",

‎packages/abnormal_security/docs/README.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -498,22 +498,22 @@ An example event for `threat` looks as following:
498498
}
499499
},
500500
"agent": {
501-
"ephemeral_id": "b66f399f-ba1c-4fe5-af82-9ca7a0204545",
502-
"id": "e2eadaf0-613d-41d9-913c-96125e06487a",
503-
"name": "elastic-agent-55334",
501+
"ephemeral_id": "900a737b-86e9-4b31-8902-9e933e02c4bc",
502+
"id": "16312af4-ae1e-4ca5-855f-6cb7e433a5a4",
503+
"name": "docker-fleet-agent",
504504
"type": "filebeat",
505505
"version": "8.13.0"
506506
},
507507
"data_stream": {
508508
"dataset": "abnormal_security.threat",
509-
"namespace": "45319",
509+
"namespace": "81591",
510510
"type": "logs"
511511
},
512512
"ecs": {
513513
"version": "8.11.0"
514514
},
515515
"elastic_agent": {
516-
"id": "e2eadaf0-613d-41d9-913c-96125e06487a",
516+
"id": "16312af4-ae1e-4ca5-855f-6cb7e433a5a4",
517517
"snapshot": false,
518518
"version": "8.13.0"
519519
},
@@ -541,7 +541,7 @@ An example event for `threat` looks as following:
541541
],
542542
"dataset": "abnormal_security.threat",
543543
"id": "2260288475997441000",
544-
"ingested": "2024-08-23T05:40:07Z",
544+
"ingested": "2024-09-03T05:30:00Z",
545545
"kind": "enrichment",
546546
"original": "{\"abxMessageId\":2260288475997441000,\"abxPortalUrl\":\"https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654\",\"attachmentCount\":0,\"attachmentNames\":[],\"attackStrategy\":\"Unknown Sender\",\"attackType\":\"Spam\",\"attackVector\":\"Link\",\"attackedParty\":\"Employee (Other)\",\"autoRemediated\":true,\"ccEmails\":[],\"fromAddress\":\"john@example.com\",\"fromName\":\"john\",\"impersonatedParty\":\"None / Others\",\"internetMessageId\":\"\\u003cAZz8NUMEST-qmuz77_koic@example\\u003e\",\"isRead\":false,\"postRemediated\":false,\"receivedTime\":\"2024-07-17T23:25:38Z\",\"recipientAddress\":\"bob@example.com\",\"remediationStatus\":\"Auto-Remediated\",\"remediationTimestamp\":\"2024-07-17T23:25:45.73564Z\",\"replyToEmails\":[],\"returnPath\":\"bounce-bob_H181S7GUCF@example.com\",\"senderDomain\":\"example.com\",\"senderIpAddress\":\"81.2.69.142\",\"sentTime\":\"2024-07-17T23:25:29Z\",\"subject\":\"YoU.have.𝗪𝟬0𝗡𝗡 a K0baIt 215-piece_ToooI_Set_Noo0wW..#GBOB\",\"summaryInsights\":[\"Abnormal Email Body HTML\",\"Invisible characters found in Email\",\"Suspicious Link\",\"Unusual Sender\",\"Unusual Sender Domain\"],\"threatId\":\"bf255f2d-a2ad-3f50-5075-fdcc24308bbd\",\"toAddresses\":[\"bob@example.com\"],\"urlCount\":1,\"urls\":[\"https://www.example.com/\"]}",
547547
"reference": "https://portal.abnormalsecurity.com/home/threat-center/remediation-history/3456765434567654",

‎packages/abnormal_security/manifest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: 3.2.1
22
name: abnormal_security
33
title: Abnormal Security
4-
version: 0.1.1
4+
version: 0.1.2
55
description: Collect logs from Abnormal Security with Elastic Agent.
66
type: integration
77
categories:

‎packages/aws_bedrock/changelog.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
- version: "0.6.0"
2+
changes:
3+
- description: Add new field `aws_bedrock.invocation.output.completion_text` having LLM text model response. Add visualization for LLM prompt and response.
4+
type: enhancement
5+
link: https://github.com/elastic/integrations/pull/10939
16
- version: "0.5.0"
27
changes:
38
- description: Add processor to set cloud.account.name field for aws_bedrock runtime data stream.

‎packages/aws_bedrock/data_stream/invocation/_dev/test/pipeline/test-aws-bedrock.log-expected.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
},
3232
"model_id": "anthropic.claude-3-haiku-20240307-v1:0",
3333
"output": {
34+
"completion_text": "The text provided is about a bug in the Elastic Fleet's Windows data stream forwarded integration. Specifically, the text discusses an issue with the `security.yml` ingest pipeline that can turn valid IPv6 addresses into invalid ones.\n\nThe bug occurs in the following code:\n\n```yaml\n- gsub:\n field: source.ip\n pattern: \"::ffff:(\\d+\\.\\d+\\.\\d+\\.\\d+)\"\n replacement: \"\\1\"\n ignore_missing: true\n```\n\nThe issue is that the pattern `::ffff:(\\d+\\.\\d+\\.\\d+\\.\\d+)` is not anchored at the beginning of the string, which can lead to unintended transformations of valid IPv6 addresses.\n\nThe text suggests a fix by using the pattern `^::ffff:` to anchor the pattern at the beginning of the string, which would correctly transform `::ffff:192.168.1.1` into `192.168.1.1` while leaving `fe80::ffff:ffff:fffe` untouched.\n\nIn summary, this is a bug report for the Elastic Fleet's Windows data stream forwarded integration, and the text provides a suggested fix for the issue.",
3435
"output_body_json": [
3536
{
3637
"message": {

‎packages/aws_bedrock/data_stream/invocation/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,78 @@ processors:
442442
ctx.aws_bedrock.invocation.input.remove("input_body_json");
443443
ctx.gen_ai.remove("prompt");
444444
}
445+
446+
- script:
447+
description: Convert chat completion response in map or list format to user friendly format
448+
lang: painless
449+
source: |
450+
try {
451+
def completionText = new StringBuilder();
452+
if (ctx.aws_bedrock?.invocation?.output?.output_body_json != null) {
453+
if (ctx.aws_bedrock.invocation.output.output_body_json instanceof List) {
454+
for (def block : ctx.aws_bedrock.invocation.output.output_body_json) {
455+
if (!(block instanceof Map)) continue;
456+
457+
if (block.containsKey('delta') && block.delta instanceof Map && block.delta.containsKey('text')) {
458+
// Titan model response
459+
completionText.append(block.delta.text);
460+
} else if (block.containsKey('outputs') && block.outputs instanceof List) {
461+
// Mistral model response
462+
for (def cont : block.outputs) {
463+
if (cont instanceof Map && cont.containsKey('text')) {
464+
completionText.append(cont.text);
465+
}
466+
}
467+
} else if (block.containsKey('generation')) {
468+
// Llama3 model response
469+
completionText.append(block.generation);
470+
} else if (block.containsKey('outputText')) {
471+
// Titan Text G1 - Express response
472+
completionText.append(block.outputText);
473+
}
474+
}
475+
}else if (ctx.aws_bedrock.invocation.output.output_body_json instanceof Map) {
476+
def block = ctx.aws_bedrock.invocation.output.output_body_json;
477+
if (block instanceof Map) {
478+
Map blockMap = (Map) block;
479+
if (blockMap.containsKey('output') && blockMap.output instanceof Map) {
480+
Map outputMap = (Map) blockMap.output;
481+
if (outputMap.containsKey('message') && outputMap.message instanceof Map) {
482+
Map messageMap = (Map) outputMap.message;
483+
if (messageMap.containsKey('content') && messageMap.content instanceof List) {
484+
List contentList = (List) messageMap.content;
485+
for (def cont : contentList) {
486+
if (cont.containsKey('text')) {
487+
completionText.append(cont.text);
488+
}
489+
}
490+
}
491+
}
492+
}
493+
}
494+
}else if (ctx.aws_bedrock.invocation.output.output_body_json instanceof String){
495+
completionText.append(ctx.aws_bedrock.invocation.output.output_body_json)
496+
}
497+
}
498+
499+
// Ensure ctx.aws_bedrock.invocation.output is initialized
500+
if (ctx.aws_bedrock?.invocation?.output == null) {
501+
ctx.aws_bedrock.invocation.output = new HashMap();
502+
}
503+
// Trim completionText if it exceeds 32766 characters
504+
if (completionText.length() > 32766) {
505+
completionText.setLength(32766);
506+
}
507+
ctx.aws_bedrock.invocation.output.completion_text = completionText.toString();
508+
}
509+
catch (Exception e) {
510+
if (ctx.aws_bedrock?.invocation?.output == null) {
511+
ctx.aws_bedrock.invocation.output = new HashMap();
512+
}
513+
ctx.aws_bedrock.invocation.output.completion_text = '';
514+
ctx.error.message = e.getMessage()
515+
}
516+
445517
- script:
446518
description: Remove massive output bodies and keep response perf stats.
447519
if: ctx.gen_ai?.completion instanceof String

‎packages/aws_bedrock/data_stream/invocation/fields/fields.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@
2020
type: keyword
2121
- name: output.output_token_count
2222
type: long
23+
- name: output.completion_text
24+
type: text
25+
description: The formatted LLM text model responses. Only a limited number of LLM text models are supported.
2326
- name: request_id
2427
type: keyword
2528
- name: schema_type

‎packages/aws_bedrock/docs/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ list log events from the specified log group.
157157
| aws_bedrock.invocation.input.input_content_type | | keyword |
158158
| aws_bedrock.invocation.input.input_token_count | todo | long |
159159
| aws_bedrock.invocation.model_id | | keyword |
160+
| aws_bedrock.invocation.output.completion_text | The formatted LLM text model responses. Only a limited number of LLM text models are supported. | text |
160161
| aws_bedrock.invocation.output.output_body_json | | flattened |
161162
| aws_bedrock.invocation.output.output_body_s3_path | | keyword |
162163
| aws_bedrock.invocation.output.output_content_type | | keyword |

0 commit comments

Comments
 (0)