[Packetbeat] Add url.extension to Packetbeat HTTP events (#25999)

legoguy1000 · mergify-bot · commit 7fbda4a54314 · 2021-06-15T16:26:58.000Z
* #25990: Add `url.extension` to Packetbeat HTTP events * update changelog * add tests * updated per comment (cherry picked from commit 3d341a8)
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -658,6 +658,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Packetbeat*
 
 
+- Add an example to packetbeat.yml of using the `forwarded` tag to disable
+  `host` metadata fields when processing network data from network tap or mirror
+  port. {pull}19209[19209]
+- Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167]
+- Add 100-continue support {issue}15830[15830] {pull}19349[19349]
+- Add initial SIP protocol support {pull}21221[21221]
+- Add support for overriding the published index on a per-protocol/flow basis. {pull}22134[22134]
+- Change build process for x-pack distribution {pull}21979[21979]
+- Tuned the internal queue size to reduce the chances of events being dropped. {pull}22650[22650]
+- Add support for "http.request.mime_type" and "http.response.mime_type". {pull}22940[22940]
+- Upgrade to ECS 1.8.0. {pull}23783[23783]
+- Add `event.type: [connection]` to flow events and include `end` for final flows. {pull}24564[24564]
+- Add `url.extension` to HTTP events {issue}25990[25990] {pull}25999[25999]
 
 *Functionbeat*
 
diff --git a/packetbeat/protos/http/event.go b/packetbeat/protos/http/event.go
@@ -90,6 +90,12 @@ func newURL(host string, port int64, path, query string) *ecs.Url {
 	if port != 80 {
 		u.Port = port
 	}
+	if path != "" {
+		periodIndex := strings.LastIndex(path, ".")
+		if periodIndex != -1 && periodIndex < len(path) {
+			u.Extension = path[(periodIndex + 1):]
+		}
+	}
 	u.Full = synthesizeFullURL(u, port)
 	return u
 }
diff --git a/packetbeat/protos/http/http_test.go b/packetbeat/protos/http/http_test.go
@@ -1845,6 +1845,57 @@ func TestHttpParser_hostHeader(t *testing.T) {
 	}
 }
 
+func TestHttpParser_Extension(t *testing.T) {
+	template := "HEAD %s HTTP/1.1\r\n" +
+		"Host: abc.com\r\n" +
+		"\r\n"
+	var store eventStore
+	http := httpModForTests(&store)
+	for _, test := range []struct {
+		title, path string
+		expected    common.MapStr
+	}{
+		{
+			title: "Zip Extension",
+			path:  "/files.zip",
+			expected: common.MapStr{
+				"url.full":      "http://abc.com/files.zip",
+				"url.extension": "zip",
+			},
+		},
+		{
+			title: "No Extension",
+			path:  "/files",
+			expected: common.MapStr{
+				"url.full":      "http://abc.com/files",
+				"url.extension": nil,
+			},
+		},
+	} {
+		t.Run(test.title, func(t *testing.T) {
+			request := fmt.Sprintf(template, test.path)
+			tcptuple := testCreateTCPTuple()
+			packet := protos.Packet{Payload: []byte(request)}
+			private := protos.ProtocolData(&httpConnectionData{})
+			private = http.Parse(&packet, tcptuple, 1, private)
+			http.Expired(tcptuple, private)
+			trans := expectTransaction(t, &store)
+			if !assert.NotNil(t, trans) {
+				t.Fatal("nil transaction")
+			}
+			for field, expected := range test.expected {
+				actual, err := trans.GetValue(field)
+				assert.Equal(t, expected, actual, field)
+				if expected != nil {
+					assert.Nil(t, err, field)
+				} else {
+					assert.Equal(t, common.ErrKeyNotFound, err, field)
+				}
+			}
+		})
+	}
+}
+
 func benchmarkHTTPMessage(b *testing.B, data []byte) {
 	http := httpModForTests(nil)
 	parser := newParser(&http.parserConfig)

Original file line number	Diff line number	Diff line change
`@@ -90,6 +90,12 @@ func newURL(host string, port int64, path, query string) *ecs.Url {`
`90`	`90`	`if port != 80 {`
`91`	`91`	`u.Port = port`
`92`	`92`	`}`
	`93`	`+ if path != "" {`
	`94`	`+ periodIndex := strings.LastIndex(path, ".")`
	`95`	`+ if periodIndex != -1 && periodIndex < len(path) {`
	`96`	`+ u.Extension = path[(periodIndex + 1):]`
	`97`	`+ }`
	`98`	`+ }`
`93`	`99`	`u.Full = synthesizeFullURL(u, port)`
`94`	`100`	`return u`
`95`	`101`	`}`