[Filebeat] Add Base64 encoded HMAC & UUID template functions to httpjson input (#27873)

legoguy1000 · marc-gr · mergify-bot · commit c3230134ef42 · 2021-09-20T11:05:46.000Z
* Init commit * Update docs and tests * Add Base64 HMAC * Remove EdgeGrid Library * Add UUID and Base64 HMAC template functions * don't upgrade google/uuid library * Format value_tpl.go Co-authored-by: Marc Guasch <marc.guasch@elastic.co> (cherry picked from commit 942d537)
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -434,6 +434,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `ignore_empty_value` flag to `httpjson` `split` processor. {pull}27880[27880]
 - Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879]
 - Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937]
+- Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc
@@ -202,11 +202,13 @@ Some built-in helper functions are provided to work with the input state inside
 - `add`: adds a list of integers and returns their sum.
 - `mul`: multiplies two integers.
 - `div`: does the integer division of two integer values.
-- `hmac`: calculates the hmac signature of a list of strings concatenated together. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
+- `hmac`: calculates the hmac signature of a list of strings concatenated together. Returns a hex encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
 - `base64Encode`: Joins and base64 encodes all supplied strings. Example `[[base64Encode "string1" "string2"]]`
 - `base64EncodeNoPad`: Joins and base64 encodes all supplied strings without padding. Example `[[base64EncodeNoPad "string1" "string2"]]`
 - `join`: joins a list of strings using the specified separator. Example: `[[join .body.arr ","]]`
 - `sprintf`: formats according to a format specifier and returns the resulting string. Refer to https://pkg.go.dev/fmt#Sprintf[the Go docs] for usage. Example: `[[sprintf "%d:%q" 34 "quote this"]]`
+- `hmacBase64`: calculates the hmac signature of a list of strings concatenated together. Returns a base64 encoded signature. Supports sha1 or sha256. Example `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`
+- `uuid`: returns a random UUID such as `a11e8780-e3e7-46d0-8e76-f66e75acf019` Example: `[[ uuid ]]`
 
 In addition to the provided functions, any of the native functions for https://golang.org/pkg/time/#Time[`time.Time`], https://golang.org/pkg/net/http/#Header[`http.Header`], and https://golang.org/pkg/net/url/#Values[`url.Values`] types can be used on the corresponding objects. Examples: `[[(now).Day]]`, `[[.last_response.header.Get "key"]]`
 
diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl.go
@@ -21,6 +21,8 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/google/uuid"
+
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
@@ -55,11 +57,13 @@ func (t *valueTpl) Unpack(in string) error {
 			"add":                 add,
 			"mul":                 mul,
 			"div":                 div,
-			"hmac":                hmacString,
+			"hmac":                hmacStringHex,
 			"base64Encode":        base64Encode,
 			"base64EncodeNoPad":   base64EncodeNoPad,
 			"join":                strings.Join,
 			"sprintf":             fmt.Sprintf,
+			"hmacBase64":          hmacStringBase64,
+			"uuid":                uuidString,
 		}).
 		Delims(leftDelim, rightDelim).
 		Parse(in)
@@ -267,10 +271,9 @@ func base64EncodeNoPad(values ...string) string {
 	return base64.RawStdEncoding.EncodeToString([]byte(data))
 }
 
-func hmacString(hmacType string, hmacKey string, values ...string) string {
-	data := strings.Join(values[:], "")
+func hmacString(hmacType string, hmacKey string, data string) []byte {
 	if data == "" {
-		return ""
+		return nil
 	}
 	// Create a new HMAC by defining the hash type and the key (as byte array)
 	var mac hash.Hash
@@ -281,11 +284,40 @@ func hmacString(hmacType string, hmacKey string, values ...string) string {
 		mac = hmac.New(sha1.New, []byte(hmacKey))
 	default:
 		// Upstream config validation prevents this from happening.
-		return ""
+		return nil
 	}
 	// Write Data to it
 	mac.Write([]byte(data))
 
+	// Get result and encode as bytes
+	return mac.Sum(nil)
+}
+
+func hmacStringHex(hmacType string, hmacKey string, values ...string) string {
+	data := strings.Join(values[:], "")
+	if data == "" {
+		return ""
+	}
+	bytes := hmacString(hmacType, hmacKey, data)
 	// Get result and encode as hexadecimal string
-	return hex.EncodeToString(mac.Sum(nil))
+	return hex.EncodeToString(bytes)
+}
+
+func hmacStringBase64(hmacType string, hmacKey string, values ...string) string {
+	data := strings.Join(values[:], "")
+	if data == "" {
+		return ""
+	}
+	bytes := hmacString(hmacType, hmacKey, data)
+
+	// Get result and encode as hexadecimal string
+	return base64.StdEncoding.EncodeToString(bytes)
+}
+
+func uuidString() string {
+	uuid, err := uuid.NewRandom()
+	if err != nil {
+		return ""
+	}
+	return uuid.String()
 }
diff --git a/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go b/x-pack/filebeat/input/httpjson/internal/v2/value_tpl_test.go
@@ -253,14 +253,14 @@ func TestValueTpl(t *testing.T) {
 			expectedVal: "4",
 		},
 		{
-			name:        "func sha1 hmac",
+			name:        "func sha1 hmac Hex",
 			value:       `[[hmac "sha1" "secret" "string1" "string2"]]`,
 			paramCtx:    emptyTransformContext(),
 			paramTr:     transformable{},
 			expectedVal: "87eca1e7cba012b2dd4a907c2ad4345a252a38f4",
 		},
 		{
-			name:        "func sha256 hmac",
+			name:        "func sha256 hmac Hex",
 			setup:       func() { timeNow = func() time.Time { return time.Unix(1627697597, 0).UTC() } },
 			teardown:    func() { timeNow = time.Now },
 			value:       `[[hmac "sha256" "secret" "string1" "string2" (formatDate (now) "RFC1123")]]`,
@@ -269,7 +269,7 @@ func TestValueTpl(t *testing.T) {
 			expectedVal: "adc61cd206e146f2d1337504e760ea70f3d2e34bedf28d07802e0e776568a06b",
 		},
 		{
-			name:          "func invalid hmac",
+			name:          "func invalid hmac Hex",
 			value:         `[[hmac "md5" "secret" "string1" "string2"]]`,
 			paramCtx:      emptyTransformContext(),
 			paramTr:       transformable{},
@@ -331,6 +331,30 @@ func TestValueTpl(t *testing.T) {
 			paramTr:     transformable{},
 			expectedVal: `"foo,bar":1`,
 		},
+		{
+			name:        "func sha1 hmac Base64",
+			value:       `[[hmacBase64 "sha1" "secret" "string1" "string2"]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "h+yh58ugErLdSpB8KtQ0WiUqOPQ=",
+		},
+		{
+			name:        "func sha256 hmac Base64",
+			setup:       func() { timeNow = func() time.Time { return time.Unix(1627697597, 0).UTC() } },
+			teardown:    func() { timeNow = time.Now },
+			value:       `[[hmacBase64 "sha256" "secret" "string1" "string2"]]`,
+			paramCtx:    emptyTransformContext(),
+			paramTr:     transformable{},
+			expectedVal: "HlglO6yRZs0Ts3MjmgnRKtTJk3fr9nt8LmeliVKZyAA=",
+		},
+		{
+			name:          "func invalid hmac Base64",
+			value:         `[[hmacBase64 "md5" "secret" "string1" "string2"]]`,
+			paramCtx:      emptyTransformContext(),
+			paramTr:       transformable{},
+			expectedVal:   "",
+			expectedError: errEmptyTemplateResult.Error(),
+		},
 	}
 
 	for _, tc := range cases {
