Skip to content

Commit eb2dc26

Browse files
andrewvcandrewvc
authored
[Heartbeat] Add Additional ECS tls.* fields (elastic#17687)
Work in support of elastic/uptime#161 This patch adds additional ECS [TLS](https://www.elastic.co/guide/en/ecs/current/ecs-tls.html) and [x509](elastic/ecs#762) fields. Note that we are blocked on the x509 fields which are not yet merged into ECS. Sample output of the `tls.*` fields with this patch is below. Note the somewhat strange nesting of data in `issuer` and `subject`. This is per the ECS spec, but a bit awkward. We may want to break this data out into the more specific ECS `x509` type in the future. For UI work we are likely fine to parse this on the client and display the CN section in most cases. I did break out the CN into its own field in `x509.subject/issuer.common_name`. However, if we do want to aggregate on issuer in the future it's good to have the full distinguished name to do that on. This PR also refactors some `libbeat` code around parsing TLS versions and adds test coverage there as well. ```json { "tls": { "certificate_not_valid_after": "2020-07-16T03:15:39Z", "certificate_not_valid_before": "2019-08-16T01:40:25Z", "server": { "hash": { "sha1": "b7b4b89ef0d0caf39d223736f0fdbb03c7b426f1", "sha256": "12b00d04db0db8caa302bfde043e88f95baceb91e86ac143e93830b4bbec726d" }, "x509": { "issuer": { "common_name": "GlobalSign CloudSSL CA - SHA256 - G3", "distinguished_name": "CN=GlobalSign CloudSSL CA - SHA256 - G3,O=GlobalSign nv-sa,C=BE" }, "not_after": "2020-07-16T03:15:39Z", "not_before": "2019-08-16T01:40:25Z", "public_key_algorithm": "RSA", "public_key_size": 2048, "serial_number": "26610543540289562361990401194", "signature_algorithm": "SHA256-RSA", "subject": { "common_name": "r2.shared.global.fastly.net", "distinguished_name": "CN=r2.shared.global.fastly.net,O=Fastly\\, Inc.,L=San Francisco,ST=California,C=US" } } } } } ``` ## How to test this PR locally Run against TLS/Non-TLS endpoints
1 parent 7f037bf commit eb2dc26

22 files changed

Lines changed: 1155 additions & 242 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
279279
*Heartbeat*
280280

281281
- Allow a list of status codes for HTTP checks. {pull}15587[15587]
282-
282+
- Add additional ECS compatible fields for TLS information. {pull}17687[17687]
283283

284284
*Journalbeat*
285285

‎heartbeat/_meta/fields.common.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,19 @@
1717
type: keyword
1818
description: >
1919
The monitors configured name
20+
multi_fields:
21+
- name: text
22+
type: text
23+
analyzer: simple
2024

2125
- name: id
2226
type: keyword
2327
description: >
2428
The monitors full job ID as used by heartbeat.
29+
multi_fields:
30+
- name: text
31+
type: text
32+
analyzer: simple
2533

2634
- name: duration
2735
type: group

‎heartbeat/docs/fields.asciidoc‎

Lines changed: 199 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,13 @@ type: keyword
215215
216216
--
217217
218+
*`monitor.name.text`*::
219+
+
220+
--
221+
type: text
222+
223+
--
224+
218225
*`monitor.id`*::
219226
+
220227
--
@@ -225,6 +232,13 @@ type: keyword
225232
226233
--
227234
235+
*`monitor.id.text`*::
236+
+
237+
--
238+
type: text
239+
240+
--
241+
228242
[float]
229243
=== duration
230244
@@ -7824,7 +7838,10 @@ TLS layer related fields.
78247838
*`tls.certificate_not_valid_before`*::
78257839
+
78267840
--
7827-
Earliest time at which the connection's certificates are valid.
7841+
7842+
deprecated:[7.8.0]
7843+
7844+
Deprecated in favor of `tls.server.x509.not_before`. Earliest time at which the connection's certificates are valid.
78287845
78297846
type: date
78307847
@@ -7833,7 +7850,10 @@ type: date
78337850
*`tls.certificate_not_valid_after`*::
78347851
+
78357852
--
7836-
Latest time at which the connection's certificates are valid.
7853+
7854+
deprecated:[7.8.0]
7855+
7856+
Deprecated in favor of `tls.server.x509.not_after`. Latest time at which the connection's certificates are valid.
78377857
78387858
type: date
78397859
@@ -7862,3 +7882,180 @@ type: long
78627882
78637883
--
78647884
7885+
[float]
7886+
=== server
7887+
7888+
Detailed x509 certificate metadata
7889+
7890+
7891+
7892+
*`tls.server.x509.alternative_names`*::
7893+
+
7894+
--
7895+
List of subject alternative names (SAN). Name types vary by certificate authority and certificate type but commonly contain IP addresses, DNS names (and wildcards), and email addresses.
7896+
7897+
type: keyword
7898+
7899+
example: *.elastic.co
7900+
7901+
--
7902+
7903+
7904+
*`tls.server.x509.issuer.common_name`*::
7905+
+
7906+
--
7907+
List of common name (CN) of issuing certificate authority.
7908+
7909+
type: keyword
7910+
7911+
example: DigiCert SHA2 High Assurance Server CA
7912+
7913+
--
7914+
7915+
*`tls.server.x509.issuer.common_name.text`*::
7916+
+
7917+
--
7918+
type: wildcard
7919+
7920+
--
7921+
7922+
*`tls.server.x509.issuer.distinguished_name`*::
7923+
+
7924+
--
7925+
Distinguished name (DN) of issuing certificate authority.
7926+
7927+
type: keyword
7928+
7929+
example: C=US, O=DigiCert Inc, OU=www.digicert.com, CN=DigiCert SHA2 High Assurance Server CA
7930+
7931+
--
7932+
7933+
*`tls.server.x509.not_after`*::
7934+
+
7935+
--
7936+
Time at which the certificate is no longer considered valid.
7937+
7938+
type: date
7939+
7940+
example: 2020-07-16 03:15:39
7941+
7942+
--
7943+
7944+
*`tls.server.x509.not_before`*::
7945+
+
7946+
--
7947+
Time at which the certificate is first considered valid.
7948+
7949+
type: date
7950+
7951+
example: 2019-08-16 01:40:25
7952+
7953+
--
7954+
7955+
*`tls.server.x509.public_key_algorithm`*::
7956+
+
7957+
--
7958+
Algorithm used to generate the public key.
7959+
7960+
type: keyword
7961+
7962+
example: RSA
7963+
7964+
--
7965+
7966+
*`tls.server.x509.public_key_curve`*::
7967+
+
7968+
--
7969+
The curve used by the elliptic curve public key algorithm. This is algorithm specific.
7970+
7971+
type: keyword
7972+
7973+
example: nistp521
7974+
7975+
--
7976+
7977+
*`tls.server.x509.public_key_exponent`*::
7978+
+
7979+
--
7980+
Exponent used to derive the public key. This is algorithm specific.
7981+
7982+
type: long
7983+
7984+
example: 65537
7985+
7986+
--
7987+
7988+
*`tls.server.x509.public_key_size`*::
7989+
+
7990+
--
7991+
The size of the public key space in bits.
7992+
7993+
type: long
7994+
7995+
example: 2048
7996+
7997+
--
7998+
7999+
*`tls.server.x509.serial_number`*::
8000+
+
8001+
--
8002+
Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters.
8003+
8004+
type: keyword
8005+
8006+
example: 55FBB9C7DEBF09809D12CCAA
8007+
8008+
--
8009+
8010+
*`tls.server.x509.signature_algorithm`*::
8011+
+
8012+
--
8013+
Identifier for certificate signature algorithm. Recommend using names found in Go Lang Crypto library (See https://github.com/golang/go/blob/go1.14/src/crypto/x509/x509.go#L337-L353).
8014+
8015+
type: keyword
8016+
8017+
example: SHA256-RSA
8018+
8019+
--
8020+
8021+
8022+
*`tls.server.x509.subject.subject.common_name`*::
8023+
+
8024+
--
8025+
List of common names (CN) of subject.
8026+
8027+
type: keyword
8028+
8029+
example: r2.shared.global.fastly.net
8030+
8031+
--
8032+
8033+
*`tls.server.x509.subject.subject.common_name.text`*::
8034+
+
8035+
--
8036+
type: wildcard
8037+
8038+
--
8039+
8040+
*`tls.server.x509.subject.subject.distinguished_name`*::
8041+
+
8042+
--
8043+
Distinguished name (DN) of the certificate subject entity.
8044+
8045+
type: keyword
8046+
8047+
example: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=r2.shared.global.fastly.net
8048+
8049+
--
8050+
8051+
*`tls.server.x509.version_number`*::
8052+
+
8053+
--
8054+
Version of x509 format.
8055+
8056+
type: keyword
8057+
8058+
example: 3
8059+
8060+
--
8061+

‎heartbeat/hbtest/hbtestutil.go‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,23 @@
1818
package hbtest
1919

2020
import (
21+
"crypto/tls"
2122
"crypto/x509"
2223
"fmt"
2324
"io"
2425
"io/ioutil"
26+
"net"
2527
"net/http"
2628
"net/http/httptest"
2729
"net/url"
2830
"os"
2931
"strconv"
3032
"strings"
3133
"testing"
34+
"time"
35+
36+
"github.com/elastic/beats/v7/heartbeat/monitors/active/dialchain/tlsmeta"
37+
"github.com/elastic/beats/v7/libbeat/common"
3238

3339
"github.com/elastic/beats/v7/heartbeat/hbtestllext"
3440

@@ -107,13 +113,25 @@ func ServerPort(server *httptest.Server) (uint16, error) {
107113

108114
// TLSChecks validates the given x509 cert at the given position.
109115
func TLSChecks(chainIndex, certIndex int, certificate *x509.Certificate) validator.Validator {
110-
return lookslike.MustCompile(map[string]interface{}{
111-
"tls": map[string]interface{}{
112-
"rtt.handshake.us": isdef.IsDuration,
113-
"certificate_not_valid_before": certificate.NotBefore,
114-
"certificate_not_valid_after": certificate.NotAfter,
115-
},
116-
})
116+
expected := common.MapStr{}
117+
// This function is well tested independently, so we just test that things match up here.
118+
tlsmeta.AddTLSMetadata(expected, tls.ConnectionState{
119+
Version: tls.VersionTLS13,
120+
HandshakeComplete: true,
121+
CipherSuite: tls.TLS_AES_128_GCM_SHA256,
122+
ServerName: certificate.Subject.CommonName,
123+
PeerCertificates: []*x509.Certificate{certificate},
124+
}, time.Duration(1))
125+
126+
expected.Put("tls.rtt.handshake.us", isdef.IsDuration)
127+
128+
return lookslike.MustCompile(expected)
129+
}
130+
131+
func TLSCertChecks(certificate *x509.Certificate) validator.Validator {
132+
expected := common.MapStr{}
133+
tlsmeta.AddCertMetadata(expected, []*x509.Certificate{certificate})
134+
return lookslike.MustCompile(expected)
117135
}
118136

119137
// BaseChecks creates a skima.Validator that represents the "monitor" field present
@@ -196,6 +214,14 @@ func ErrorChecks(msgSubstr string, errType string) validator.Validator {
196214
})
197215
}
198216

217+
func ExpiredCertChecks(cert *x509.Certificate) validator.Validator {
218+
msg := x509.CertificateInvalidError{Cert: cert, Reason: x509.Expired}.Error()
219+
return lookslike.Compose(
220+
ErrorChecks(msg, "io"),
221+
TLSCertChecks(cert),
222+
)
223+
}
224+
199225
// RespondingTCPChecks creates a skima.Validator that represents the "tcp" field present
200226
// in all heartbeat events that use a Tcp connection as part of their DialChain
201227
func RespondingTCPChecks() validator.Validator {
@@ -215,3 +241,23 @@ func CertToTempFile(t *testing.T, cert *x509.Certificate) *os.File {
215241
certFile.WriteString(x509util.CertToPEMString(cert))
216242
return certFile
217243
}
244+
245+
func StartHTTPSServer(t *testing.T, tlsCert tls.Certificate) (host string, port string, cert *x509.Certificate, doClose func() error) {
246+
cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
247+
require.NoError(t, err)
248+
249+
// No need to start a real server, since this is invalid, we just
250+
l, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
251+
Certificates: []tls.Certificate{tlsCert},
252+
})
253+
require.NoError(t, err)
254+
255+
srv := &http.Server{Handler: HelloWorldHandler(200)}
256+
go func() {
257+
srv.Serve(l)
258+
}()
259+
260+
host, port, err = net.SplitHostPort(l.Addr().String())
261+
require.NoError(t, err)
262+
return host, port, cert, srv.Close
263+
}

‎heartbeat/include/fields.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)