Skip to content

Commit be5576a

Browse files
jsorianojsoriano
authored
Merge branch 'master' into mongodb-json-format
2 parents 0769350 + 2871d29 commit be5576a

239 files changed

Lines changed: 4403 additions & 2283 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ci/packaging.groovy‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,8 @@ pipeline {
208208
'packetbeat',
209209
'x-pack/auditbeat',
210210
'x-pack/dockerlogbeat',
211-
'x-pack/elastic-agent',
211+
// See https://github.com/elastic/beats/issues/26239
212+
// 'x-pack/elastic-agent',
212213
'x-pack/filebeat',
213214
'x-pack/heartbeat',
214215
'x-pack/metricbeat',

‎.ci/packer_cache.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ function dockerPullImages() {
4545
docker.elastic.co/kibana/kibana:${SNAPSHOT}
4646
docker.elastic.co/logstash/logstash:${SNAPSHOT}
4747
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
48+
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
49+
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
4850
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
4951
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
5052
docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main

‎.go-version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.16.4
1+
1.16.5

‎CHANGELOG-developer.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,3 +116,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
116116
- Update Go version to 1.15.12. {pull}25629[25629]
117117
- Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
118118
- Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]
119+
- Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]

‎CHANGELOG.next.asciidoc‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
106106
- Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
107107

108108
*Heartbeat*
109+
- Add support for screenshot blocks and use newer synthetics flags that only works in newer synthetics betas. {pull}25808[25808]
109110

110111
*Journalbeat*
111112

@@ -131,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
131132
- Add support for the MemoryPressure, DiskPressure, OutOfDisk and PIDPressure status conditions in state_node. {pull}23905[23905]
132133
- Remove xpack enabled flag on ES, Logstash, Beats and Kibana {pull}24427[24427]
133134
- Adjust host fields to adopt new names from 1.9.0 ECS. {pull}24312[24312]
135+
- Add replicas.ready field to state_statefulset in Kubernetes module{pull}26088[26088]
134136

135137
*Packetbeat*
136138

@@ -238,6 +240,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
238240
- Fix 'make setup' instructions for a new beat {pull}24944[24944]
239241
- Fix out of date FreeBSD vagrantbox. {pull}25652[25652]
240242
- Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
243+
- Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
244+
- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
245+
- In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively.
241246

242247
*Auditbeat*
243248

@@ -259,6 +264,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
259264
- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
260265
- system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
261266
- Note incompatibility of system/socket on ARM. {pull}23381[23381]
267+
- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
262268

263269
*Filebeat*
264270

@@ -275,6 +281,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
275281
- Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
276282
- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
277283
- Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
284+
- Fix default config template values for paths on oracle module: {pull}26276[26276]
285+
- Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
278286

279287
*Filebeat*
280288

@@ -381,6 +389,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
381389
- Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
382390
- Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
383391
- Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
392+
- Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
393+
- Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
394+
- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
384395

385396
*Heartbeat*
386397

@@ -488,6 +499,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
488499
- Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
489500
- Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
490501
- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
502+
- Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
491503

492504
*Packetbeat*
493505

@@ -585,6 +597,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
585597
- Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
586598
- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422]
587599
- Improve ES output error insights. {pull}25825[25825]
600+
- Libbeat: report beat version to monitoring. {pull}26214[26214]
588601

589602
*Auditbeat*
590603

@@ -807,6 +820,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
807820
- Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
808821
- Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
809822
- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774]
823+
- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
824+
- Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168]
825+
- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
826+
- Make `filestream` input GA. {pull}26127[26127]
827+
- Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
828+
- Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
829+
- Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
830+
- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
831+
- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
810832

811833
*Heartbeat*
812834

@@ -939,6 +961,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
939961
- Add additional network metrics to docker/network {pull}25354[25354]
940962
- Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
941963
- Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
964+
- Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
965+
- Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
966+
- Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
942967

943968
*Packetbeat*
944969

‎Vagrantfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,7 @@ Vagrant.configure("2") do |config|
307307
c.vm.provision "shell", inline: $unixProvision, privileged: false
308308
c.vm.provision "shell", inline: $freebsdShellUpdate, privileged: true
309309
c.vm.provision "shell", inline: gvmProvision(arch="amd64", os="freebsd"), privileged: false
310+
c.vm.provision "shell", inline: "sudo mount -t linprocfs /dev/null /proc", privileged: false
310311
end
311312

312313
# OpenBSD 6.0

‎auditbeat/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.16.4
1+
FROM golang:1.16.5
22

33
RUN \
44
apt-get update \

‎auditbeat/module/auditd/audit_linux.go‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ const (
5151

5252
lostEventsUpdateInterval = time.Second * 15
5353
maxDefaultStreamBufferConsumers = 4
54+
55+
setPIDMaxRetries = 5
5456
)
5557

5658
type backpressureStrategy uint8
@@ -137,10 +139,32 @@ func newAuditClient(c *Config, log *logp.Logger) (*libaudit.AuditClient, error)
137139
return libaudit.NewAuditClient(nil)
138140
}
139141

142+
func closeAuditClient(client *libaudit.AuditClient) error {
143+
discard := func(bytes []byte) ([]syscall.NetlinkMessage, error) {
144+
return nil, nil
145+
}
146+
// Drain the netlink channel in parallel to Close() to prevent a deadlock.
147+
// This goroutine will terminate once receive from netlink errors (EBADF,
148+
// EBADFD, or any other error). This happens because the fd is closed.
149+
go func() {
150+
for {
151+
_, err := client.Netlink.Receive(true, discard)
152+
switch err {
153+
case nil, syscall.EINTR:
154+
case syscall.EAGAIN:
155+
time.Sleep(50 * time.Millisecond)
156+
default:
157+
return
158+
}
159+
}
160+
}()
161+
return client.Close()
162+
}
163+
140164
// Run initializes the audit client and receives audit messages from the
141165
// kernel until the reporter's done channel is closed.
142166
func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
143-
defer ms.client.Close()
167+
defer closeAuditClient(ms.client)
144168

145169
if err := ms.addRules(reporter); err != nil {
146170
reporter.Error(err)
@@ -164,7 +188,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
164188
go func() {
165189
defer func() { // Close the most recently allocated "client" instance.
166190
if client != nil {
167-
client.Close()
191+
closeAuditClient(client)
168192
}
169193
}()
170194
timer := time.NewTicker(lostEventsUpdateInterval)
@@ -178,7 +202,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
178202
ms.updateKernelLostMetric(status.Lost)
179203
} else {
180204
ms.log.Error("get status request failed:", err)
181-
if err = client.Close(); err != nil {
205+
if err = closeAuditClient(client); err != nil {
182206
ms.log.Errorw("Error closing audit monitoring client", "error", err)
183207
}
184208
client, err = libaudit.NewAuditClient(nil)
@@ -233,7 +257,7 @@ func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error {
233257
if err != nil {
234258
return errors.Wrap(err, "failed to create audit client for adding rules")
235259
}
236-
defer client.Close()
260+
defer closeAuditClient(client)
237261

238262
// Don't attempt to change configuration if audit rules are locked (enabled == 2).
239263
// Will result in EPERM.
@@ -350,10 +374,12 @@ func (ms *MetricSet) initClient() error {
350374
return errors.Wrap(err, "failed to enable auditing in the kernel")
351375
}
352376
}
377+
353378
if err := ms.client.WaitForPendingACKs(); err != nil {
354379
return errors.Wrap(err, "failed to wait for ACKs")
355380
}
356-
if err := ms.client.SetPID(libaudit.WaitForReply); err != nil {
381+
382+
if err := ms.setPID(setPIDMaxRetries); err != nil {
357383
if errno, ok := err.(syscall.Errno); ok && errno == syscall.EEXIST && status.PID != 0 {
358384
return fmt.Errorf("failed to set audit PID. An audit process is already running (PID %d)", status.PID)
359385
}
@@ -362,6 +388,20 @@ func (ms *MetricSet) initClient() error {
362388
return nil
363389
}
364390

391+
func (ms *MetricSet) setPID(retries int) (err error) {
392+
if err = ms.client.SetPID(libaudit.WaitForReply); err == nil || errors.Cause(err) != syscall.ENOBUFS || retries == 0 {
393+
return err
394+
}
395+
// At this point the netlink channel is congested (ENOBUFS).
396+
// Drain and close the client, then retry with a new client.
397+
closeAuditClient(ms.client)
398+
if ms.client, err = newAuditClient(&ms.config, ms.log); err != nil {
399+
return errors.Wrapf(err, "failed to recover from ENOBUFS")
400+
}
401+
ms.log.Info("Recovering from ENOBUFS ...")
402+
return ms.setPID(retries - 1)
403+
}
404+
365405
func (ms *MetricSet) updateKernelLostMetric(lost uint32) {
366406
if !ms.kernelLost.enabled {
367407
return

‎dev-tools/mage/crossbuild.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,15 +197,22 @@ func crossBuildImage(platform string) (string, error) {
197197
tagSuffix := "main"
198198

199199
switch {
200-
case strings.HasPrefix(platform, "darwin"):
200+
case platform == "darwin/amd64":
201201
tagSuffix = "darwin-debian10"
202-
case strings.HasPrefix(platform, "linux/armv7"):
203-
tagSuffix = "armhf"
204-
case strings.HasPrefix(platform, "linux/arm"):
202+
case platform == "darwin/arm64":
203+
tagSuffix = "darwin-arm64-debian10"
204+
case platform == "linux/arm64":
205205
tagSuffix = "arm"
206+
// when it runs on a ARM64 host/worker.
206207
if runtime.GOARCH == "arm64" {
207208
tagSuffix = "base-arm-debian9"
208209
}
210+
case platform == "linux/armv5":
211+
tagSuffix = "armel"
212+
case platform == "linux/armv6":
213+
tagSuffix = "armel"
214+
case platform == "linux/armv7":
215+
tagSuffix = "armhf"
209216
case strings.HasPrefix(platform, "linux/mips"):
210217
tagSuffix = "mips"
211218
case strings.HasPrefix(platform, "linux/ppc"):

‎filebeat/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.16.4
1+
FROM golang:1.16.5
22

33
RUN \
44
apt-get update \

0 commit comments

Comments
 (0)