Skip to content

Commit 20ef0f0

Browse files
ChrsMarkChrsMark
committed
Merge remote-tracking branch 'upstream/master' into pod-node-watcher
2 parents 2d83a37 + 28d00c7 commit 20ef0f0

29 files changed

Lines changed: 1229 additions & 1186 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
2727
- add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
2828
- Use data streams instead of indices for storing events from Beats. {pull}28450[28450]
2929
- Remove option `setup.template.type` and always load composable template with data streams. {pull}28450[28450]
30-
- Remove several ILM options (`rollover_alias` and `pattern`) as data streams does not require index aliases. {pull}28450[28450]
30+
- Remove several ILM options (`rollover_alias` and `pattern`) as data streams does not require index aliases. {pull}28450[28450]
3131
- Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
3232
- Remove deprecated `--template` and `--ilm-policy` flags. Use `--index-management` instead. {pull}28870[28870]
3333
- Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927]
@@ -147,6 +147,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
147147
- Skip configuration checks in autodiscover for configurations that are already running {pull}29048[29048]
148148
- Fix `decode_json_processor` to always respect `add_error_key` {pull}29107[29107]
149149
- Fix `add_labels` flattening of array values. {pull}29211[29211]
150+
- Overwrite index name in index template correctly. {issue}28571[28571] {pull}29299[29299]
150151

151152
*Auditbeat*
152153

@@ -195,6 +196,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
195196
- Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
196197
- Fix `threatintel.misp` filters configuration. {issue}27970[27970]
197198
- Fix opening files on Windows in filestream so open files can be deleted. {issue}29113[29113] {pull}29180[29180]
199+
- Fix handling of escaped newlines in the `decode_cef` processor. {issue}16995[16995] {pull}29268[29268]
198200
- Fix `panw` module ingest errors for GLOBALPROTECT logs {pull}29154[29154]
199201

200202
*Heartbeat*
@@ -356,6 +358,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
356358
- Add support in aws-s3 input for custom script parsing of s3 notifications. {pull}28946[28946]
357359
- Improve error handling in aws-s3 input for malformed s3 notifications. {issue}28828[28828] {pull}28946[28946]
358360
- Add support for parsers on journald input {pull}29070[29070]
361+
- Add elapsed time information to `aws-s3` input errors and log messages. {pull}29328[29328]
359362
- Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087]
360363

361364
*Heartbeat*

‎libbeat/common/transport/tlscommon/tls_config.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ type TLSConfig struct {
7979

8080
// CATrustedFingerprint is the HEX encoded fingerprint of a CA certificate. If present in the chain
8181
// this certificate will be added to the list of trusted CAs (RootCAs) during the handshake.
82-
CATrustedFingerprint string `config:"ca_trusted_fingerprint" yaml:"ca_trusted_fingerprint,omitempty"`
82+
CATrustedFingerprint string
8383

8484
// time returns the current time as the number of seconds since the epoch.
8585
// If time is nil, TLS uses time.Now.
@@ -159,16 +159,19 @@ func (c *TLSConfig) BuildServerConfig(host string) *tls.Config {
159159
}
160160

161161
func trustRootCA(cfg *TLSConfig, peerCerts []*x509.Certificate) error {
162+
logger := logp.NewLogger("tls")
163+
logger.Info("'ca_trusted_fingerprint' set, looking for matching fingerprints")
162164
fingerprint, err := hex.DecodeString(cfg.CATrustedFingerprint)
163165
if err != nil {
164-
return fmt.Errorf("decode fingerprint: %w", err)
166+
return fmt.Errorf("decode 'ca_trusted_fingerprint': %w", err)
165167
}
166168

167169
for _, cert := range peerCerts {
168170
// Compute digest for each certificate.
169171
digest := sha256.Sum256(cert.Raw)
170172

171173
if bytes.Equal(digest[0:], fingerprint) {
174+
logger.Info("CA certificate matching 'ca_trusted_fingerprint' found, adding it to 'certificate_authorities'")
172175
// Make sure the fingerprint matches a CA certificate
173176
if cert.IsCA {
174177
if cfg.RootCAs == nil {
@@ -181,7 +184,7 @@ func trustRootCA(cfg *TLSConfig, peerCerts []*x509.Certificate) error {
181184
}
182185
}
183186

184-
logp.NewLogger("tls").Warn("no CA certificate matching the fingerprint")
187+
logger.Warn("no CA certificate matching the fingerprint")
185188
return nil
186189
}
187190

‎libbeat/dashboards/modify_json.go‎

Lines changed: 6 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,6 @@ import (
2323
"fmt"
2424
"regexp"
2525

26-
"github.com/pkg/errors"
27-
2826
"github.com/elastic/beats/v7/libbeat/common"
2927
"github.com/elastic/beats/v7/libbeat/logp"
3028
)
@@ -53,44 +51,14 @@ func ReplaceIndexInIndexPattern(index string, content common.MapStr) (err error)
5351
return nil
5452
}
5553

56-
list, ok := content["objects"]
57-
if !ok {
58-
return errors.New("empty index pattern")
59-
}
60-
61-
updateObject := func(obj common.MapStr) {
62-
// This uses Put instead of DeepUpdate to avoid modifying types for
63-
// inner objects. (DeepUpdate will replace maps with MapStr).
64-
obj.Put("id", index)
65-
// Only overwrite title if it exists.
66-
if _, err := obj.GetValue("attributes.title"); err == nil {
67-
obj.Put("attributes.title", index)
68-
}
54+
// This uses Put instead of DeepUpdate to avoid modifying types for
55+
// inner objects. (DeepUpdate will replace maps with MapStr).
56+
content.Put("id", index)
57+
// Only overwrite title if it exists.
58+
if _, err := content.GetValue("attributes.title"); err == nil {
59+
content.Put("attributes.title", index)
6960
}
7061

71-
switch v := list.(type) {
72-
case []interface{}:
73-
for _, objIf := range v {
74-
switch obj := objIf.(type) {
75-
case common.MapStr:
76-
updateObject(obj)
77-
case map[string]interface{}:
78-
updateObject(obj)
79-
default:
80-
return errors.Errorf("index pattern object has unexpected type %T", v)
81-
}
82-
}
83-
case []map[string]interface{}:
84-
for _, obj := range v {
85-
updateObject(obj)
86-
}
87-
case []common.MapStr:
88-
for _, obj := range v {
89-
updateObject(obj)
90-
}
91-
default:
92-
return errors.Errorf("index pattern objects have unexpected type %T", v)
93-
}
9462
return nil
9563
}
9664

‎libbeat/dashboards/modify_json_test.go‎

Lines changed: 27 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -124,118 +124,68 @@ func TestReplaceIndexInIndexPattern(t *testing.T) {
124124
index string
125125
expected common.MapStr
126126
}{
127-
{
128-
title: "Replace in []interface(map).map",
129-
input: common.MapStr{"objects": []interface{}{map[string]interface{}{
130-
"id": "phonybeat-*",
131-
"type": "index-pattern",
132-
"attributes": map[string]interface{}{
133-
"title": "phonybeat-*",
134-
"timeFieldName": "@timestamp",
135-
}}}},
136-
index: "otherindex-*",
137-
expected: common.MapStr{"objects": []interface{}{map[string]interface{}{
138-
"id": "otherindex-*",
139-
"type": "index-pattern",
140-
"attributes": map[string]interface{}{
141-
"title": "otherindex-*",
142-
"timeFieldName": "@timestamp",
143-
}}}},
144-
},
145-
{
146-
title: "Replace in []interface(map).mapstr",
147-
input: common.MapStr{"objects": []interface{}{map[string]interface{}{
148-
"id": "phonybeat-*",
149-
"type": "index-pattern",
150-
"attributes": common.MapStr{
151-
"title": "phonybeat-*",
152-
"timeFieldName": "@timestamp",
153-
}}}},
154-
index: "otherindex-*",
155-
expected: common.MapStr{"objects": []interface{}{map[string]interface{}{
156-
"id": "otherindex-*",
157-
"type": "index-pattern",
158-
"attributes": common.MapStr{
159-
"title": "otherindex-*",
160-
"timeFieldName": "@timestamp",
161-
}}}},
162-
},
163-
{
164-
title: "Replace in []map.mapstr",
165-
input: common.MapStr{"objects": []map[string]interface{}{{
166-
"id": "phonybeat-*",
167-
"type": "index-pattern",
168-
"attributes": common.MapStr{
169-
"title": "phonybeat-*",
170-
"timeFieldName": "@timestamp",
171-
}}}},
172-
index: "otherindex-*",
173-
expected: common.MapStr{"objects": []map[string]interface{}{{
174-
"id": "otherindex-*",
175-
"type": "index-pattern",
176-
"attributes": common.MapStr{
177-
"title": "otherindex-*",
178-
"timeFieldName": "@timestamp",
179-
}}}},
180-
},
181127
{
182128
title: "Replace in []mapstr.mapstr",
183-
input: common.MapStr{"objects": []common.MapStr{{
129+
input: common.MapStr{
184130
"id": "phonybeat-*",
185131
"type": "index-pattern",
186132
"attributes": common.MapStr{
187133
"title": "phonybeat-*",
188134
"timeFieldName": "@timestamp",
189-
}}}},
135+
}},
190136
index: "otherindex-*",
191-
expected: common.MapStr{"objects": []common.MapStr{{
137+
expected: common.MapStr{
192138
"id": "otherindex-*",
193139
"type": "index-pattern",
194140
"attributes": common.MapStr{
195141
"title": "otherindex-*",
196142
"timeFieldName": "@timestamp",
197-
}}}},
143+
}},
198144
},
199145
{
200146
title: "Replace in []mapstr.interface(mapstr)",
201-
input: common.MapStr{"objects": []common.MapStr{{
147+
input: common.MapStr{
202148
"id": "phonybeat-*",
203149
"type": "index-pattern",
204150
"attributes": interface{}(common.MapStr{
205151
"title": "phonybeat-*",
206152
"timeFieldName": "@timestamp",
207-
})}}},
153+
})},
208154
index: "otherindex-*",
209-
expected: common.MapStr{"objects": []common.MapStr{{
155+
expected: common.MapStr{
210156
"id": "otherindex-*",
211157
"type": "index-pattern",
212158
"attributes": interface{}(common.MapStr{
213159
"title": "otherindex-*",
214160
"timeFieldName": "@timestamp",
215-
})}}},
161+
})},
216162
},
217163
{
218164
title: "Do not create missing attributes",
219-
input: common.MapStr{"objects": []common.MapStr{{
220-
"id": "phonybeat-*",
221-
"type": "index-pattern",
222-
}}},
165+
input: common.MapStr{
166+
"attributes": common.MapStr{},
167+
"id": "phonybeat-*",
168+
"type": "index-pattern",
169+
},
223170
index: "otherindex-*",
224-
expected: common.MapStr{"objects": []common.MapStr{{
225-
"id": "otherindex-*",
226-
"type": "index-pattern",
227-
}}},
171+
expected: common.MapStr{
172+
"attributes": common.MapStr{},
173+
"id": "otherindex-*",
174+
"type": "index-pattern",
175+
},
228176
},
229177
{
230178
title: "Create missing id",
231-
input: common.MapStr{"objects": []common.MapStr{{
232-
"type": "index-pattern",
233-
}}},
179+
input: common.MapStr{
180+
"attributes": common.MapStr{},
181+
"type": "index-pattern",
182+
},
234183
index: "otherindex-*",
235-
expected: common.MapStr{"objects": []common.MapStr{{
236-
"id": "otherindex-*",
237-
"type": "index-pattern",
238-
}}},
184+
expected: common.MapStr{
185+
"attributes": common.MapStr{},
186+
"id": "otherindex-*",
187+
"type": "index-pattern",
188+
},
239189
},
240190
}
241191

‎libbeat/docs/release-notes/breaking/breaking-7.16.asciidoc‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
[[breaking-changes-7.16]]
2+
3+
=== Breaking changes in 7.16
4+
++++
5+
<titleabbrev>7.16</titleabbrev>
6+
++++
7+
8+
See the <<release-notes,release notes>> for a complete list of changes,
9+
including changes to beta or experimental functionality.
10+
11+
//NOTE: The notable-breaking-changes tagged regions are re-used in the
12+
//Installation and Upgrade Guide
13+
14+
// tag::notable-breaking-changes[]
15+
16+
[discrete]
17+
==== {journalbeat} is removed in 7.16
18+
19+
{journalbeat}, a lightweight shipper for collecting logs written by the Journald
20+
system service, is removed in 7.16. This functionality is instead provided as
21+
a {filebeat} input. If you're currently using {journalbeat}, you should
22+
use the `journald` input in {filebeat} instead. For more information, refer to
23+
the
24+
{filebeat-ref}/filebeat-input-journald.html[Journald input] documentation.
25+
26+
If you're using {agent} instead of {beats}, you can collect Journald logs by
27+
adding the *Custom Journald logs* integration to your agent policy. For more
28+
information, refer to
29+
{fleet-guide}/add-integration-to-policy.html[Add an {agent} integration to a policy].
30+
31+
//TODO: Add pointer to the integrations docs for custom journald logs when
32+
//available.
33+
34+
[discrete]
35+
==== Custom {beats} generator is deprecated in 7.16
36+
37+
The generator code for creating custom {beats} is deprecated in 7.16.0 and will
38+
be removed in 8.0.0. You can continue to build custom {beats} using the
39+
generators available in 7.16, or refer to existing {beats} as working examples.
40+
41+
// end::notable-breaking-changes[]
42+

‎libbeat/docs/release-notes/breaking/breaking.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ changes, but there are breaking changes between major versions (e.g. 6.x to
1111

1212
See the following topics for a description of breaking changes:
1313

14+
* <<breaking-changes-7.16>>
15+
1416
* <<breaking-changes-7.15>>
1517

1618
* <<breaking-changes-7.14>>
@@ -43,6 +45,8 @@ See the following topics for a description of breaking changes:
4345

4446
* <<breaking-changes-7.0>>
4547

48+
include::breaking-7.16.asciidoc[]
49+
4650
include::breaking-7.15.asciidoc[]
4751

4852
include::breaking-7.14.asciidoc[]

‎libbeat/outputs/outest/batch.go‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ const (
4141
BatchRetry
4242
BatchRetryEvents
4343
BatchCancelled
44-
BatchCancelledEvents
4544
)
4645

4746
func NewBatch(in ...beat.Event) *Batch {
@@ -76,10 +75,6 @@ func (b *Batch) Cancelled() {
7675
b.doSignal(BatchSignal{Tag: BatchCancelled})
7776
}
7877

79-
func (b *Batch) CancelledEvents(events []publisher.Event) {
80-
b.doSignal(BatchSignal{Tag: BatchCancelledEvents, Events: events})
81-
}
82-
8378
func (b *Batch) doSignal(sig BatchSignal) {
8479
b.Signals = append(b.Signals, sig)
8580
if b.OnSignal != nil {

‎libbeat/processors/add_kubernetes_metadata/config.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,12 @@ type PluginConfig []map[string]common.Config
5252

5353
func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig {
5454
return kubeAnnotatorConfig{
55-
SyncPeriod: 10 * time.Minute,
56-
CleanupTimeout: 60 * time.Second,
57-
DefaultMatchers: Enabled{true},
58-
DefaultIndexers: Enabled{true},
59-
Scope: "node",
55+
SyncPeriod: 10 * time.Minute,
56+
CleanupTimeout: 60 * time.Second,
57+
DefaultMatchers: Enabled{true},
58+
DefaultIndexers: Enabled{true},
59+
Scope: "node",
60+
AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
6061
}
6162
}
6263

0 commit comments

Comments
 (0)