[Windows] Add additional fields to windows package system_operational integration by Mikaayenson · Pull Request #5120 · elastic/integrations

Mikaayenson · 2023-01-26T19:40:03Z

What does this PR do?

Summary

Adds the field winlog.event_data.TargetProcessGUID of type keyword to sysmon_operational.

DATA example

Details

{
  "is_partial" : false,
  "is_running" : false,
  "took" : 66,
  "timed_out" : false,
  "hits" : {
    "total" : {
      "value" : 1,
      "relation" : "eq"
    },
    "sequences" : [
      {
        "join_keys" : [
          "747f3d96-68a7-43f1-8cbe-e8d6dadd0358",
          "{747f3d96-feac-6174-0000-0010fb029a00}",
          "{747f3d96-feae-6174-0000-00106ad39a00}"
        ],
        "events" : [
          {
            "_index" : ".ds-logs-windows.sysmon_operational-default-2021.10.23-000001",
            "_id" : "7RECsXwBE6i8IeLQ59M9",
            "_source" : {
              "agent" : {
                "hostname" : "MSEDGEWIN10",
                "name" : "MSEDGEWIN10",
                "id" : "b79edb1f-7b68-4a66-8a9e-c679030c1d8e",
                "ephemeral_id" : "d085494b-52ea-4845-a4b1-46381c2243c7",
                "type" : "filebeat",
                "version" : "7.15.0"
              },
              "process" : {
                "args" : [
                  """C:\Windows\System32\WerFault.exe"""
                ],
                "parent" : {
                  "args" : [
                    """C:\Program Files\Microsoft Office\Office14\WINWORD.EXE""",
                    "/n",
                    """C:\Users\IEUser\Desktop\119dfd04194220b4e6aba006f0a3c4ae268ad3245fe196152697a4315b43fe07.doc"""
                  ],
                  "name" : "WINWORD.EXE",
                  "pid" : 25492,
                  "args_count" : 3,
                  "entity_id" : "{747f3d96-feac-6174-0000-0010fb029a00}",
                  "executable" : """C:\Program Files\Microsoft Office\Office14\WINWORD.EXE""",
                  "command_line" : "\"C:\\Program Files\\Microsoft Office\\Office14\\WINWORD.EXE\" /n \"C:\\Users\\IEUser\\Desktop\\119dfd04194220b4e6aba006f0a3c4ae268ad3245fe196152697a4315b43fe07.doc\""
                },
                "pe" : {
                  "file_version" : "10.0.17763.1339 (WinBuild.160101.0800)",
                  "product" : "Microsoft® Windows® Operating System",
                  "imphash" : "bfc763a7169835eecb91919badb76b57",
                  "description" : "Windows Problem Reporting",
                  "original_file_name" : "WerFault.exe",
                  "company" : "Microsoft Corporation"
                },
                "name" : "WerFault.exe",
                "pid" : 24700,
                "working_directory" : """C:\Users\IEUser\Desktop\""",
                "args_count" : 1,
                "entity_id" : "{747f3d96-feae-6174-0000-00106ad39a00}",
                "hash" : {
                  "sha1" : "c6a79796df32ada80fa4e166fc1ca45b73b61dc0",
                  "sha256" : "dc9adb75c9e3cb860619cd3c76544fde47f3839de775d9a9ed410f1e0d6e765a",
                  "md5" : "f9e990b0acd986721c49f5f89db32eec"
                },
                "executable" : """C:\Windows\System32\WerFault.exe""",
                "command_line" : "\"C:\\Windows\\System32\\WerFault.exe\""
              },
              "winlog" : {
                "computer_name" : "MSEDGEWIN10",
                "process" : {
                  "pid" : 3428,
                  "thread" : {
                    "id" : 4688
                  }
                },
                "channel" : "Microsoft-Windows-Sysmon/Operational",
                "event_data" : {
                  "Company" : "Microsoft Corporation",
                  "LogonGuid" : "{747f3d96-f9b3-6174-0000-00207edf3900}",
                  "Description" : "Windows Problem Reporting",
                  "IntegrityLevel" : "Medium",
                  "TerminalSessionId" : "1",
                  "FileVersion" : "10.0.17763.1339 (WinBuild.160101.0800)",
                  "Product" : "Microsoft® Windows® Operating System",
                  "LogonId" : "0x39df7e"
                },
                "opcode" : "Info",
                "version" : 5,
                "record_id" : "561649",
                "task" : "Process Create (rule: ProcessCreate)",
                "event_id" : "1",
                "provider_guid" : "{5770385f-c22a-43e0-bf4c-06f5698ffbd9}",
                "api" : "wineventlog",
                "provider_name" : "Microsoft-Windows-Sysmon",
                "user" : {
                  "identifier" : "S-1-5-18",
                  "domain" : "NT AUTHORITY",
                  "name" : "SYSTEM",
                  "type" : "User"
                }
              },
              "log" : {
                "level" : "information"
              },
              "elastic_agent" : {
                "id" : "b79edb1f-7b68-4a66-8a9e-c679030c1d8e",
                "version" : "7.15.0",
                "snapshot" : false
              },
              "message" : "Process Create:\nRuleName: \nUtcTime: 2021-10-24 06:35:26.449\nProcessGuid: {747f3d96-feae-6174-0000-00106ad39a00}\nProcessId: 24700\nImage: C:\\Windows\\System32\\WerFault.exe\nFileVersion: 10.0.17763.1339 (WinBuild.160101.0800)\nDescription: Windows Problem Reporting\nProduct: Microsoft® Windows® Operating System\nCompany: Microsoft Corporation\nOriginalFileName: WerFault.exe\nCommandLine: \"C:\\Windows\\System32\\WerFault.exe\"\nCurrentDirectory: C:\\Users\\IEUser\\Desktop\\\nUser: MSEDGEWIN10\\IEUser\nLogonGuid: {747f3d96-f9b3-6174-0000-00207edf3900}\nLogonId: 0x39DF7E\nTerminalSessionId: 1\nIntegrityLevel: Medium\nHashes: SHA1=C6A79796DF32ADA80FA4E166FC1CA45B73B61DC0,MD5=F9E990B0ACD986721C49F5F89DB32EEC,SHA256=DC9ADB75C9E3CB860619CD3C76544FDE47F3839DE775D9A9ED410F1E0D6E765A,IMPHASH=BFC763A7169835EECB91919BADB76B57\nParentProcessGuid: {747f3d96-feac-6174-0000-0010fb029a00}\nParentProcessId: 25492\nParentImage: C:\\Program Files\\Microsoft Office\\Office14\\WINWORD.EXE\nParentCommandLine: \"C:\\Program Files\\Microsoft Office\\Office14\\WINWORD.EXE\" /n \"C:\\Users\\IEUser\\Desktop\\116dfd041942c0b4e6aba006f0a3c4ae268ad3245fe196152697a3315b43fe07.doc\"",
              "input" : {
                "type" : "winlog"
              },
              "@timestamp" : "2021-10-24T06:35:26.449Z",
              "ecs" : {
                "version" : "1.12.0"
              },
              "related" : {
                "user" : [
                  "IEUser"
                ],
                "hash" : [
                  "dc9adb75c9e3cb860619cd3c76544fde47f3839de775d9a9ed410f1e0d6e765a",
                  "c6a79796df32ada80fa4e166fc1ca45b73b61dc0",
                  "f9e990b0acd986721c49f5f89db32eec",
                  "bfc763a7169835eecb91919badb76b57"
                ]
              },
              "data_stream" : {
                "namespace" : "default",
                "type" : "logs",
                "dataset" : "windows.sysmon_operational"
              },
              "host" : {
                "hostname" : "MSEDGEWIN10",
                "os" : {
                  "build" : "17763.1397",
                  "kernel" : "10.0.17763.1397 (WinBuild.160101.0800)",
                  "name" : "Windows 10 Enterprise Evaluation",
                  "type" : "windows",
                  "family" : "windows",
                  "version" : "10.0",
                  "platform" : "windows"
                },
                "ip" : [
                  "fe80::c50d:519f:96a4:e108",
                  "10.0.2.15",
                  "fe80::530:1baa:ea3a:2240",
                  "10.0.3.15"
                ],
                "name" : "MSEDGEWIN10",
                "id" : "747f3d96-68a7-43f1-8cbe-e8d6dadd0358",
                "mac" : [
                  "08:00:27:e6:e5:59",
                  "08:00:27:6b:89:4e"
                ],
                "architecture" : "x86_64"
              },
              "event" : {
                "agent_id_status" : "verified",
                "ingested" : "2021-10-24T06:35:29Z",
                "code" : "1",
                "provider" : "Microsoft-Windows-Sysmon",
                "created" : "2021-10-24T06:35:27.362Z",
                "kind" : "event",
                "action" : "Process Create (rule: ProcessCreate)",
                "category" : [
                  "process"
                ],
                "type" : [
                  "start"
                ],
                "dataset" : "windows.sysmon_operational"
              },
              "user" : {
                "domain" : "MSEDGEWIN10",
                "name" : "IEUser",
                "id" : "S-1-5-18"
              }
            }
          },
          {
            "_index" : ".ds-logs-windows.sysmon_operational-default-2021.10.23-000001",
            "_id" : "7hECsXwBE6i8IeLQ59M9",
            "_source" : {
              "agent" : {
                "hostname" : "MSEDGEWIN10",
                "name" : "MSEDGEWIN10",
                "id" : "b79edb1f-7b68-4a66-8a9e-c679030c1d8e",
                "type" : "filebeat",
                "ephemeral_id" : "d085494b-52ea-4845-a4b1-46381c2243c7",
                "version" : "7.15.0"
              },
              "process" : {
                "name" : "WINWORD.EXE",
                "pid" : 25492,
                "thread" : {
                  "id" : 23924
                },
                "entity_id" : "{747f3d96-feac-6174-0000-0010fb029a00}",
                "executable" : """C:\Program Files\Microsoft Office\Office14\WINWORD.EXE"""
              },
              "winlog" : {
                "computer_name" : "MSEDGEWIN10",
                "process" : {
                  "pid" : 3428,
                  "thread" : {
                    "id" : 4688
                  }
                },
                "channel" : "Microsoft-Windows-Sysmon/Operational",
                "event_data" : {
                  "GrantedAccess" : "0x1fffff",
                  "TargetProcessId" : "24700",
                  "TargetImage" : """C:\Windows\System32\WerFault.exe""",
                  "CallTrace" : """C:\Windows\SYSTEM32\ntdll.dll+a0fb4|C:\Windows\System32\KERNELBASE.dll+485d2|C:\Windows\System32\KERNELBASE.dll+45eaa|C:\Windows\System32\KERNELBASE.dll+45a36|C:\Windows\System32\KERNEL32.DLL+1c153|UNKNOWN(00007FFE11374AAE)""",
                  "TargetProcessGUID" : "{747f3d96-feae-6174-0000-00106ad39a00}"
                },
                "opcode" : "Info",
                "version" : 3,
                "record_id" : "561650",
                "event_id" : "10",
                "task" : "Process accessed (rule: ProcessAccess)",
                "provider_guid" : "{5770385f-c22a-43e0-bf4c-06f5698ffbd9}",
                "api" : "wineventlog",
                "provider_name" : "Microsoft-Windows-Sysmon",
                "user" : {
                  "identifier" : "S-1-5-18",
                  "domain" : "NT AUTHORITY",
                  "name" : "SYSTEM",
                  "type" : "User"
                }
              },
              "log" : {
                "level" : "information"
              },
              "elastic_agent" : {
                "id" : "b79edb1f-7b68-4a66-8a9e-c679030c1d8e",
                "version" : "7.15.0",
                "snapshot" : false
              },
              "message" : """Process accessed:
RuleName: 
UtcTime: 2021-10-24 06:35:26.460
SourceProcessGUID: {747f3d96-feac-6174-0000-0010fb029a00}
SourceProcessId: 25492
SourceThreadId: 23924
SourceImage: C:\Program Files\Microsoft Office\Office14\WINWORD.EXE
TargetProcessGUID: {747f3d96-feae-6174-0000-00106ad39a00}
TargetProcessId: 24700
TargetImage: C:\Windows\System32\WerFault.exe
GrantedAccess: 0x1FFFFF
CallTrace: C:\Windows\SYSTEM32\ntdll.dll+a0fb4|C:\Windows\System32\KERNELBASE.dll+485d2|C:\Windows\System32\KERNELBASE.dll+45eaa|C:\Windows\System32\KERNELBASE.dll+45a36|C:\Windows\System32\KERNEL32.DLL+1c153|UNKNOWN(00007FFE11374AAE)""",
              "input" : {
                "type" : "winlog"
              },
              "@timestamp" : "2021-10-24T06:35:26.460Z",
              "ecs" : {
                "version" : "1.12.0"
              },
              "data_stream" : {
                "namespace" : "default",
                "type" : "logs",
                "dataset" : "windows.sysmon_operational"
              },
              "host" : {
                "hostname" : "MSEDGEWIN10",
                "os" : {
                  "build" : "17763.1397",
                  "kernel" : "10.0.17763.1397 (WinBuild.160101.0800)",
                  "name" : "Windows 10 Enterprise Evaluation",
                  "family" : "windows",
                  "type" : "windows",
                  "version" : "10.0",
                  "platform" : "windows"
                },
                "ip" : [
                  "fe80::c50d:519f:96a4:e108",
                  "10.0.2.15",
                  "fe80::530:1baa:ea3a:2240",
                  "10.0.3.15"
                ],
                "name" : "MSEDGEWIN10",
                "id" : "747f3d96-68a7-43f1-8cbe-e8d6dadd0358",
                "mac" : [
                  "08:00:27:e6:e5:59",
                  "08:00:27:6b:89:4e"
                ],
                "architecture" : "x86_64"
              },
              "event" : {
                "agent_id_status" : "verified",
                "ingested" : "2021-10-24T06:35:29Z",
                "code" : "10",
                "provider" : "Microsoft-Windows-Sysmon",
                "created" : "2021-10-24T06:35:27.469Z",
                "kind" : "event",
                "action" : "Process accessed (rule: ProcessAccess)",
                "category" : [
                  "process"
                ],
                "type" : [
                  "access"
                ],
                "dataset" : "windows.sysmon_operational"
              },
              "user" : {
                "id" : "S-1-5-18"
              }
            }
          }
        ]
      }
    ]
  }
}

Checklist

I have reviewed tips for building integrations and this pull request is aligned with them.
I have verified that all data streams collect metrics or logs.
I have added an entry to my package's changelog.yml file.
I have verified that Kibana version constraints are current according to guidelines.

elasticmachine · 2023-01-26T19:50:56Z

💚 Build Succeeded

the below badges are clickable and redirect to their specific view in the CI or DOCS

Expand to view the summary

Build stats

Start Time: 2023-02-01T23:59:32.816+0000
Duration: 16 min 17 sec

Test stats 🧪

Test	Results
Failed	0
Passed	129
Skipped	0
Total	129

🤖 GitHub comments

Expand to view the GitHub comments

To re-run your PR in the CI, just comment with:

/test : Re-trigger the build.

Mikaayenson · 2023-01-26T19:56:56Z

Is it possible to also include these in this PR? :

- "winlog.event_data.AccessList": "keyword"
- "winlog.event_data.AttributeLDAPDisplayName": "keyword"
- "winlog.event_data.AttributeValue": "keyword"
- "winlog.event_data.CallTrace": "keyword"   # included in sample event in summary
- "winlog.event_data.ClientProcessId": "keyword"
- "winlog.event_data.GrantedAccess": "keyword" # included in sample event in summary
- "winlog.event_data.ObjectClass": "keyword"
- "winlog.event_data.ObjectDN": "keyword"
- "winlog.event_data.ParentProcessId": "keyword"
- "winlog.event_data.Properties": "keyword"
- "winlog.event_data.RelativeTargetName": "keyword"
- "winlog.event_data.ShareName": "keyword"
- "winlog.event_data.TargetImage": "keyword"  # included in sample event in summary
- "winlog.event_data.TargetProcessGUID": "keyword"  # included in sample event in summary
- "winlog.event_data.SchemaFriendlyName": "keyword"
- "winlog.event_data.Resource": "keyword"
- "winlog.event_data.ImagePath": "keyword"
- "winlog.event_data.TaskName": "keyword"
- "winlog.event_data.EnabledPrivilegeList": "keyword"

elasticmachine · 2023-01-26T19:57:14Z

🌐 Coverage report

Name	Metrics % (`covered/total`)	Diff
Packages	100.0% (`4/4`)	💚
Files	87.5% (`7/8`)	👎 -12.5
Classes	87.5% (`7/8`)	👎 -12.5
Methods	83.516% (`76/91`)	👎 -2.198
Lines	92.714% (`5039/5435`)	👎 -0.668
Conditionals	100.0% (`0/0`)	💚

efd6 · 2023-01-26T23:17:20Z

Is it possible to also include these in this PR

This seems reasonable.

Would you please add tests that cover these fields?

packages/windows/changelog.yml

packages/windows/manifest.yml

Mikaayenson · 2023-01-30T20:50:04Z

Is it possible to also include these in this PR

This seems reasonable.

Would you please add tests that cover these fields?

@efd6 Certainly. Can you guide what kind of tests and where to add them? These fields are used within our detection-rules and tracked in an internal non-ecs-schema fil, that tracks things not within the integrations to date. I'm not too familiar with the integration workflows/processes.

efd6 · 2023-01-31T00:03:19Z

Do you have events with event.original present from sysmon_operational? These JSON events can be inserted into test-events.json here https://github.com/elastic/integrations/tree/main/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline and the expectations can then be generated with elastic-package test pipeline -g run in the windows package directory.

See here for elastic-package.

Mikaayenson · 2023-01-31T03:09:49Z

Do you have events with event.original present from sysmon_operational?

I will check the original detection rules PRs that introduced these fields to see if event data was included. If not, then we may be able to include them in this PR as I did not create the original detections, and therefore do not have any generated data.

efd6 · 2023-01-31T03:11:16Z

If you are able to provide scrubbed XML documents for the events, then I can generate tests data.

Mikaayenson · 2023-01-31T17:01:59Z

If you are able to provide scrubbed XML documents for the events, then I can generate tests data.

So I think we can proceed with just these fields instead of all the rest. Just found out that those fields exist in the system.security package/integration and should not be in this integration.

"winlog.event_data.CallTrace": "keyword" # included in sample event in summary
"winlog.event_data.GrantedAccess": "keyword" # included in sample event in summary
"winlog.event_data.TargetImage": "keyword" # included in sample event in summary
"winlog.event_data.TargetProcessId": "keyword" # included in sample event in summary
"winlog.event_data.TargetProcessGUID": "keyword" # included in sample event in summary

The original event includes all of these fields.

Mikaayenson · 2023-01-31T18:32:39Z

@efd6 I added test data and updated the fields that we want to include. Do you think you can generate the expected data with everything I provided? Other than that, I think this PR is ready to go.

efd6 · 2023-01-31T21:27:45Z

Thanks.

If you apply the following changes, the tests will pass.

diff --git a/packages/windows/changelog.yml b/packages/windows/changelog.yml
index ccb4f6bd1..8cae567cd 100644
--- a/packages/windows/changelog.yml
+++ b/packages/windows/changelog.yml
@@ -1,5 +1,5 @@
 # newer versions go on top
-- version: "1.70.0"
+- version: "1.17.0"
   changes:
     - description: Add CallTrace, GrantedAccess, TargetImage, TargetProcessGUID, fields to sysmon_operational fields
       type: enhancement
diff --git a/packages/windows/data_stream/forwarded/_dev/test/pipeline/test-sysmon-operational-events.json-expected.json b/packages/windows/data_stream/forwarded/_dev/test/pipeline/test-sysmon-operational-events.json-expected.json
index 0839ad6d7..21539806c 100644
--- a/packages/windows/data_stream/forwarded/_dev/test/pipeline/test-sysmon-operational-events.json-expected.json
+++ b/packages/windows/data_stream/forwarded/_dev/test/pipeline/test-sysmon-operational-events.json-expected.json
@@ -847,8 +847,9 @@
                 ],
                 "question": {
                     "name": "confiant-integrations.global.ssl.fastly.net",
-                    "registered_domain": "confiant-integrations.global.ssl.fastly.net",
-                    "top_level_domain": "global.ssl.fastly.net"
+                    "registered_domain": "global.ssl.fastly.net",
+                    "subdomain": "confiant-integrations",
+                    "top_level_domain": "ssl.fastly.net"
                 },
                 "resolved_ip": [
                     "89.160.20.156",
@@ -2434,8 +2435,9 @@
                 ],
                 "question": {
                     "name": "clarium.freetls.fastly.net",
-                    "registered_domain": "clarium.freetls.fastly.net",
-                    "top_level_domain": "freetls.fastly.net"
+                    "registered_domain": "freetls.fastly.net",
+                    "subdomain": "clarium",
+                    "top_level_domain": "fastly.net"
                 },
                 "resolved_ip": [
                     "89.160.20.156",
diff --git a/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json b/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json
index 1faa38016..ea3c357bd 100644
--- a/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json
+++ b/packages/windows/data_stream/sysmon_operational/_dev/test/pipeline/test-events.json-expected.json
@@ -723,8 +723,9 @@
                 ],
                 "question": {
                     "name": "confiant-integrations.global.ssl.fastly.net",
-                    "registered_domain": "confiant-integrations.global.ssl.fastly.net",
-                    "top_level_domain": "global.ssl.fastly.net"
+                    "registered_domain": "global.ssl.fastly.net",
+                    "subdomain": "confiant-integrations",
+                    "top_level_domain": "ssl.fastly.net"
                 },
                 "resolved_ip": [
                     "89.160.20.156",
@@ -2322,8 +2323,9 @@
                 ],
                 "question": {
                     "name": "clarium.freetls.fastly.net",
-                    "registered_domain": "clarium.freetls.fastly.net",
-                    "top_level_domain": "freetls.fastly.net"
+                    "registered_domain": "freetls.fastly.net",
+                    "subdomain": "clarium",
+                    "top_level_domain": "fastly.net"
                 },
                 "resolved_ip": [
                     "89.160.20.156",
@@ -22101,6 +22103,135 @@
                 },
                 "version": 5
             }
+        },
+        {
+            "@timestamp": "2021-10-24T06:35:26.449Z",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "Process Create (rule: ProcessCreate)",
+                "agent_id_status": "verified",
+                "category": [
+                    "process"
+                ],
+                "code": "1",
+                "created": "2021-10-24T06:35:27.362Z",
+                "dataset": "windows.sysmon_operational",
+                "ingested": "2021-10-24T06:35:29Z",
+                "kind": "event",
+                "provider": "Microsoft-Windows-Sysmon",
+                "type": [
+                    "start"
+                ]
+            },
+            "log": {
+                "level": "information"
+            },
+            "process": {
+                "pe": {
+                    "company": "Microsoft Corporation",
+                    "description": "Windows Problem Reporting",
+                    "file_version": "10.0.17763.1339 (WinBuild.160101.0800)",
+                    "product": "Microsoft® Windows® Operating System"
+                }
+            },
+            "user": {
+                "id": "S-1-5-18"
+            },
+            "winlog": {
+                "api": "wineventlog",
+                "channel": "Microsoft-Windows-Sysmon/Operational",
+                "computer_name": "MSEDGEWIN10",
+                "event_data": {
+                    "Company": "Microsoft Corporation",
+                    "Description": "Windows Problem Reporting",
+                    "FileVersion": "10.0.17763.1339 (WinBuild.160101.0800)",
+                    "IntegrityLevel": "Medium",
+                    "LogonGuid": "{747f3d96-f9b3-6174-0000-00207edf3900}",
+                    "LogonId": "0x39df7e",
+                    "Product": "Microsoft® Windows® Operating System",
+                    "TerminalSessionId": "1"
+                },
+                "event_id": "1",
+                "opcode": "Info",
+                "process": {
+                    "pid": 3428,
+                    "thread": {
+                        "id": 4688
+                    }
+                },
+                "provider_guid": "{5770385f-c22a-43e0-bf4c-06f5698ffbd9}",
+                "provider_name": "Microsoft-Windows-Sysmon",
+                "record_id": "561649",
+                "task": "Process Create (rule: ProcessCreate)",
+                "user": {
+                    "domain": "NT AUTHORITY",
+                    "identifier": "S-1-5-18",
+                    "name": "SYSTEM",
+                    "type": "User"
+                },
+                "version": 5
+            }
+        },
+        {
+            "@timestamp": "2021-10-24T06:35:26.460Z",
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "action": "Process accessed (rule: ProcessAccess)",
+                "agent_id_status": "verified",
+                "category": [
+                    "process"
+                ],
+                "code": "10",
+                "created": "2021-10-24T06:35:27.469Z",
+                "dataset": "windows.sysmon_operational",
+                "ingested": "2021-10-24T06:35:29Z",
+                "kind": "event",
+                "provider": "Microsoft-Windows-Sysmon",
+                "type": [
+                    "access"
+                ]
+            },
+            "log": {
+                "level": "information"
+            },
+            "user": {
+                "id": "S-1-5-18"
+            },
+            "winlog": {
+                "api": "wineventlog",
+                "channel": "Microsoft-Windows-Sysmon/Operational",
+                "computer_name": "MSEDGEWIN10",
+                "event_data": {
+                    "CallTrace": "C:\\Windows\\SYSTEM32\\ntdll.dll+a0fb4|C:\\Windows\\System32\\KERNELBASE.dll+485d2|C:\\Windows\\System32\\KERNELBASE.dll+45eaa|C:\\Windows\\System32\\KERNELBASE.dll+45a36|C:\\Windows\\System32\\KERNEL32.DLL+1c153|UNKNOWN(00007FFE11374AAE)",
+                    "GrantedAccess": "0x1fffff",
+                    "TargetImage": "C:\\Windows\\System32\\WerFault.exe",
+                    "TargetProcessGUID": "{747f3d96-feae-6174-0000-00106ad39a00}",
+                    "TargetProcessId": "24700"
+                },
+                "event_id": "10",
+                "opcode": "Info",
+                "process": {
+                    "pid": 3428,
+                    "thread": {
+                        "id": 4688
+                    }
+                },
+                "provider_guid": "{5770385f-c22a-43e0-bf4c-06f5698ffbd9}",
+                "provider_name": "Microsoft-Windows-Sysmon",
+                "record_id": "561650",
+                "task": "Process accessed (rule: ProcessAccess)",
+                "user": {
+                    "domain": "NT AUTHORITY",
+                    "identifier": "S-1-5-18",
+                    "name": "SYSTEM",
+                    "type": "User"
+                },
+                "version": 3
+            }
         }
     ]
 }
\ No newline at end of file
diff --git a/packages/windows/data_stream/sysmon_operational/sample_event.json b/packages/windows/data_stream/sysmon_operational/sample_event.json
index 0e6816625..d3d8205d6 100644
--- a/packages/windows/data_stream/sysmon_operational/sample_event.json
+++ b/packages/windows/data_stream/sysmon_operational/sample_event.json
@@ -1,12 +1,11 @@
 {
     "@timestamp": "2019-07-18T03:34:01.261Z",
     "agent": {
-        "ephemeral_id": "0670a96e-1852-42bc-b667-66e022ab1c89",
-        "hostname": "docker-fleet-agent",
-        "id": "0d57cbc7-6410-455a-840c-08fd44507a26",
+        "ephemeral_id": "e90c830e-2d46-424a-8adf-c7b37b885e11",
+        "id": "ef922225-3c64-4634-8160-c93dfcee8f20",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.5.1"
     },
     "data_stream": {
         "dataset": "windows.sysmon_operational",
@@ -42,9 +41,9 @@
         "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "0d57cbc7-6410-455a-840c-08fd44507a26",
+        "id": "ef922225-3c64-4634-8160-c93dfcee8f20",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.5.1"
     },
     "event": {
         "agent_id_status": "verified",
@@ -54,7 +53,7 @@
         "code": "22",
         "created": "2019-07-18T03:34:02.025Z",
         "dataset": "windows.sysmon_operational",
-        "ingested": "2022-03-31T08:42:26Z",
+        "ingested": "2023-01-31T21:24:36Z",
         "kind": "event",
         "original": "\u003cEvent xmlns='http://schemas.microsoft.com/win/2004/08/events/event'\u003e\u003cSystem\u003e\u003cProvider Name='Microsoft-Windows-Sysmon' Guid='{5770385f-c22a-43e0-bf4c-06f5698ffbd9}'/\u003e\u003cEventID\u003e22\u003c/EventID\u003e\u003cVersion\u003e5\u003c/Version\u003e\u003cLevel\u003e4\u003c/Level\u003e\u003cTask\u003e22\u003c/Task\u003e\u003cOpcode\u003e0\u003c/Opcode\u003e\u003cKeywords\u003e0x8000000000000000\u003c/Keywords\u003e\u003cTimeCreated SystemTime='2019-07-18T03:34:02.025237700Z'/\u003e\u003cEventRecordID\u003e67\u003c/EventRecordID\u003e\u003cCorrelation/\u003e\u003cExecution ProcessID='2828' ThreadID='1684'/\u003e\u003cChannel\u003eMicrosoft-Windows-Sysmon/Operational\u003c/Channel\u003e\u003cComputer\u003evagrant-2016\u003c/Computer\u003e\u003cSecurity UserID='S-1-5-18'/\u003e\u003c/System\u003e\u003cEventData\u003e\u003cData Name='RuleName'\u003e\u003c/Data\u003e\u003cData Name='UtcTime'\u003e2019-07-18 03:34:01.261\u003c/Data\u003e\u003cData Name='ProcessGuid'\u003e{fa4a0de6-e8a9-5d2f-0000-001053699900}\u003c/Data\u003e\u003cData Name='ProcessId'\u003e2736\u003c/Data\u003e\u003cData Name='QueryName'\u003ewww.msn.com\u003c/Data\u003e\u003cData Name='QueryStatus'\u003e0\u003c/Data\u003e\u003cData Name='QueryResults'\u003etype:  5 www-msn-com.a-0003.a-msedge.net;type:  5 a-0003.a-msedge.net;::ffff:204.79.197.203;\u003c/Data\u003e\u003cData Name='Image'\u003eC:\\Program Files (x86)\\Internet Explorer\\iexplore.exe\u003c/Data\u003e\u003c/EventData\u003e\u003c/Event\u003e",
         "provider": "Microsoft-Windows-Sysmon",

Mikaayenson · 2023-02-01T03:02:57Z

Thanks.

If you apply the following changes, the tests will pass.

@efd6 Thanks for the diff. =) You made it too easy for me. Thanks for the patience and pointers.

efd6 · 2023-02-01T03:36:34Z

Sorry, I missed diffs in the README.md (these would be made by running elastic-package build from the diff to sample_event.json in the diff above).

[2023-02-01T03:12:12.149Z] --- want
[2023-02-01T03:12:12.149Z] +++ got
[2023-02-01T03:12:12.149Z] @@ -749,8 +749,7 @@
[2023-02-01T03:12:12.149Z]      "agent": {
[2023-02-01T03:12:12.149Z] -        "ephemeral_id": "0670a96e-1852-42bc-b667-66e022ab1c89",
[2023-02-01T03:12:12.149Z] -        "hostname": "docker-fleet-agent",
[2023-02-01T03:12:12.149Z] -        "id": "0d57cbc7-6410-455a-840c-08fd44507a26",
[2023-02-01T03:12:12.149Z] +        "ephemeral_id": "e90c830e-2d46-424a-8adf-c7b37b885e11",
[2023-02-01T03:12:12.149Z] +        "id": "ef922225-3c64-4634-8160-c93dfcee8f20",
[2023-02-01T03:12:12.149Z]          "name": "docker-fleet-agent",
[2023-02-01T03:12:12.149Z]          "type": "filebeat",
[2023-02-01T03:12:12.149Z] -        "version": "7.17.0"
[2023-02-01T03:12:12.149Z] +        "version": "8.5.1"
[2023-02-01T03:12:12.149Z]      },
[2023-02-01T03:12:12.149Z] @@ -790,5 +789,5 @@
[2023-02-01T03:12:12.149Z]      "elastic_agent": {
[2023-02-01T03:12:12.149Z] -        "id": "0d57cbc7-6410-455a-840c-08fd44507a26",
[2023-02-01T03:12:12.149Z] +        "id": "ef922225-3c64-4634-8160-c93dfcee8f20",
[2023-02-01T03:12:12.149Z]          "snapshot": false,
[2023-02-01T03:12:12.149Z] -        "version": "7.17.0"
[2023-02-01T03:12:12.149Z] +        "version": "8.5.1"
[2023-02-01T03:12:12.149Z]      },
[2023-02-01T03:12:12.149Z] @@ -802,3 +801,3 @@
[2023-02-01T03:12:12.149Z]          "dataset": "windows.sysmon_operational",
[2023-02-01T03:12:12.149Z] -        "ingested": "2022-03-31T08:42:26Z",
[2023-02-01T03:12:12.149Z] +        "ingested": "2023-01-31T21:24:36Z",
[2023-02-01T03:12:12.149Z]          "kind": "event",

Mikaayenson · 2023-02-01T22:43:07Z

@efd6 Do you have any recommendations on how to resolve this last issue?

efd6 · 2023-02-01T23:13:57Z

/test

efd6 · 2023-02-01T23:14:11Z

This is passing locally.

efd6 · 2023-02-01T23:47:03Z

OK. I have an answer. There is a change in stack behaviour between 8.0.0 and 8.6.0 which is causing the failure. I picked up the diff and added that into the change I posted above. I'll push the fix to this branch

Mikaayenson · 2023-02-02T00:23:40Z

OK. I have an answer. There is a change in stack behaviour between 8.0.0 and 8.6.0 which is causing the failure. I picked up the diff and added that into the change I posted above. I'll push the fix to this branch

Thanks for the help and quick responses! 🙏 Do you mind sharing how you fixed it?

efd6 · 2023-02-02T00:55:35Z

Yeah, I ran the tests on 8.0.0.

(cd ..; elastic-package stack up -v -d --version 8.0.0); eval "$(elastic-package stack shellinit)"
elastic-package test -v -g
elastic-package check
elastic-package build

Mikaayenson · 2023-02-02T01:36:49Z

Thanks again for the help! 🥳

efd6 · 2023-02-02T01:49:29Z

Sorry, note the edit above.

elasticmachine · 2023-02-02T01:58:11Z

Package windows - 1.17.0 containing this change is available at https://epr.elastic.co/search?package=windows

Add TargetLogonId to windows package system_operational integration

e78fca9

Mikaayenson added enhancement New feature or request Integration:windows Windows labels Jan 26, 2023

Mikaayenson requested a review from a team as a code owner January 26, 2023 19:40

Mikaayenson self-assigned this Jan 26, 2023

Mikaayenson requested a review from a team as a code owner January 26, 2023 19:40

Mikaayenson requested review from fearful-symmetry and rdner January 26, 2023 19:40

add pr number

8a9f0aa

Mikaayenson mentioned this pull request Jan 26, 2023

Review use of non-ecs-schema elastic/detection-rules#1776

Open

3 tasks

efd6 reviewed Jan 26, 2023

View reviewed changes

packages/windows/changelog.yml Outdated Show resolved Hide resolved

packages/windows/manifest.yml Outdated Show resolved Hide resolved

Mikaayenson changed the title ~~[Windows] Add TargetLogonId to windows package system_operational integration~~ [Windows] Add additional fields to windows package system_operational integration Jan 31, 2023

add new fieds and test data

8693215

escape data

4b2a070

Add expected test data

f37b560

update readme

a2d8794

run expect generation and elastic-package check with stack 8.0.0

d10e42f

efd6 approved these changes Feb 2, 2023

View reviewed changes

Mikaayenson merged commit ee0ae79 into main Feb 2, 2023

Conversation

Mikaayenson commented Jan 26, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

What does this PR do?

Summary

Checklist

Uh oh!

elasticmachine commented Jan 26, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

💚 Build Succeeded

Build stats

Test stats 🧪

🤖 GitHub comments

Uh oh!

Mikaayenson commented Jan 26, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

elasticmachine commented Jan 26, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

🌐 Coverage report

Uh oh!

efd6 commented Jan 26, 2023

Uh oh!

Uh oh!

Uh oh!

Mikaayenson commented Jan 30, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

efd6 commented Jan 31, 2023

Uh oh!

Mikaayenson commented Jan 31, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

efd6 commented Jan 31, 2023

Uh oh!

Mikaayenson commented Jan 31, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Mikaayenson commented Jan 31, 2023

Uh oh!

efd6 commented Jan 31, 2023

Uh oh!

Mikaayenson commented Feb 1, 2023

Uh oh!

efd6 commented Feb 1, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Mikaayenson commented Feb 1, 2023

Uh oh!

efd6 commented Feb 1, 2023

Uh oh!

efd6 commented Feb 1, 2023

Uh oh!

efd6 commented Feb 1, 2023

Uh oh!

Mikaayenson commented Feb 2, 2023

Uh oh!

efd6 commented Feb 2, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Mikaayenson commented Feb 2, 2023

Uh oh!

efd6 commented Feb 2, 2023

Uh oh!

elasticmachine commented Feb 2, 2023

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

Mikaayenson commented Jan 26, 2023 •

edited

Loading

elasticmachine commented Jan 26, 2023 •

edited

Loading

Mikaayenson commented Jan 26, 2023 •

edited

Loading

elasticmachine commented Jan 26, 2023 •

edited

Loading

Mikaayenson commented Jan 30, 2023 •

edited

Loading

Mikaayenson commented Jan 31, 2023 •

edited

Loading

Mikaayenson commented Jan 31, 2023 •

edited

Loading

efd6 commented Feb 1, 2023 •

edited

Loading

efd6 commented Feb 2, 2023 •

edited

Loading