Skip to content

Commit 788dd36

Browse files
mergify[bot]mergify[bot]
authored
Merge branch '7.x' into mergify/bp/7.x/pr-25186
2 parents 881a18d + dad84f4 commit 788dd36

25 files changed

Lines changed: 549 additions & 59 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,35 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
165165
- system/package: Fix an error that can occur while trying to persist package metadata. {issue}18536[18536] {pull}18887[18887]
166166
- system/socket: Fix dataset using 100% CPU and becoming unresponsive in some scenarios. {pull}19033[19033] {pull}19764[19764]
167167
- system/socket: Fixed tracking of long-running connections. {pull}19033[19033]
168+
- system/package: Fix librpm loading on Fedora 31/32. {pull}NNNN[NNNN]
169+
- file_integrity: Create fsnotify watcher only when starting file_integrity module {pull}19505[19505]
170+
- auditd: Fix spelling of anomaly in `event.category`.
171+
- auditd: Fix typo in `event.action` of `removed-user-role-from`. {pull}19300[19300]
172+
- auditd: Fix typo in `event.action` of `used-suspicious-link`. {pull}19300[19300]
173+
- system/socket: Fix kprobe grouping to allow running more than one instance. {pull}20325[20325]
174+
- system/socket: Fixed a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690]
175+
- file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282]
176+
- auditd: Fix an error condition causing a lot of `audit_send_reply` kernel threads being created. {pull}22673[22673]
177+
- system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
178+
- system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
179+
- Note incompatibility of system/socket on ARM. {pull}23381[23381]
180+
181+
*Filebeat*
182+
183+
- Fix mapping of fortinet.firewall.mem as integer. {pull}19335[19335]
184+
- Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766]
185+
- Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590]
186+
- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
187+
- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
188+
- Fix mapping error when zeek weird logs do not contain IP addresses. {pull}15906[15906]
189+
- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942]
190+
- Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900]
191+
- Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
192+
- Fix a connection error in httpjson input. {pull}16123[16123]
193+
- Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
194+
- Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
195+
- Strip Azure Eventhub connection string in debug logs. {pulll}25066[25066]
196+
- Fix o365 module config when client_secret contains special characters. {issue}25058[25058]
168197

169198
*Filebeat*
170199

@@ -558,6 +587,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
558587
- Handle datastreams for fleet. {pull}24223[24223]
559588
- Add --sandbox option for browser monitor. {pull}24172[24172]
560589
- Support additional 'root' fields from synthetics. {pull}24770[24770]
590+
- Browser zip_url source type. {pull}24714[24714]
561591

562592
*Heartbeat*
563593

‎filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,10 @@ filebeat.inputs:
281281
# are matching any regular expression from the list. By default, no files are dropped.
282282
#prospector.scanner.exclude_files: ['.gz$']
283283
284+
# Include files. A list of regular expressions to match. Filebeat keeps only the files that
285+
# are matching any regular expression from the list. By default, no files are dropped.
286+
#prospector.scanner.include_files: ['/var/log/.*']
287+
284288
# Expand "**" patterns into regular glob patterns.
285289
#prospector.scanner.recursive_glob: true
286290

‎filebeat/filebeat.reference.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,10 @@ filebeat.inputs:
688688
# are matching any regular expression from the list. By default, no files are dropped.
689689
#prospector.scanner.exclude_files: ['.gz$']
690690

691+
# Include files. A list of regular expressions to match. Filebeat keeps only the files that
692+
# are matching any regular expression from the list. By default, no files are dropped.
693+
#prospector.scanner.include_files: ['/var/log/.*']
694+
691695
# Expand "**" patterns into regular glob patterns.
692696
#prospector.scanner.recursive_glob: true
693697

‎filebeat/fileset/fileset.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,8 +307,11 @@ func getTemplateFunctions(vars map[string]interface{}) (template.FuncMap, error)
307307
return false
308308
},
309309
"tojson": func(v interface{}) (string, error) {
310-
bytes, err := json.Marshal(v)
311-
return string(bytes), err
310+
var buf strings.Builder
311+
enc := json.NewEncoder(&buf)
312+
enc.SetEscapeHTML(false)
313+
err := enc.Encode(v)
314+
return buf.String(), err
312315
},
313316
"IngestPipeline": func(shortID string) string {
314317
return formatPipelineID(

‎filebeat/input/filestream/fswatch.go‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ type watcherFactory func(paths []string, cfg *common.Config) (loginp.FSWatcher,
5151
type fileScanner struct {
5252
paths []string
5353
excludedFiles []match.Matcher
54+
includedFiles []match.Matcher
5455
symlinks bool
5556

5657
log *logp.Logger
@@ -234,6 +235,7 @@ func (w *fileWatcher) GetFiles() map[string]os.FileInfo {
234235

235236
type fileScannerConfig struct {
236237
ExcludedFiles []match.Matcher `config:"exclude_files"`
238+
IncludedFiles []match.Matcher `config:"include_files"`
237239
Symlinks bool `config:"symlinks"`
238240
RecursiveGlob bool `config:"recursive_glob"`
239241
}
@@ -249,6 +251,7 @@ func newFileScanner(paths []string, cfg fileScannerConfig) (loginp.FSScanner, er
249251
fs := fileScanner{
250252
paths: paths,
251253
excludedFiles: cfg.ExcludedFiles,
254+
includedFiles: cfg.IncludedFiles,
252255
symlinks: cfg.Symlinks,
253256
log: logp.NewLogger(scannerName),
254257
}
@@ -337,7 +340,7 @@ func (s *fileScanner) GetFiles() map[string]os.FileInfo {
337340
}
338341

339342
func (s *fileScanner) shouldSkipFile(file string) bool {
340-
if s.isFileExcluded(file) {
343+
if s.isFileExcluded(file) || !s.isFileIncluded(file) {
341344
s.log.Debugf("Exclude file: %s", file)
342345
return true
343346
}
@@ -359,6 +362,18 @@ func (s *fileScanner) shouldSkipFile(file string) bool {
359362
return true
360363
}
361364

365+
originalFile, err := filepath.EvalSymlinks(file)
366+
if err != nil {
367+
s.log.Debugf("finding path to original file has failed %s: %+v", file, err)
368+
return true
369+
}
370+
// Check if original file is included to make sure we are not reading from
371+
// unwanted files.
372+
if s.isFileExcluded(originalFile) || !s.isFileIncluded(originalFile) {
373+
s.log.Debugf("Exclude original file: %s", file)
374+
return true
375+
}
376+
362377
return false
363378
}
364379

@@ -384,6 +399,13 @@ func (s *fileScanner) isFileExcluded(file string) bool {
384399
return len(s.excludedFiles) > 0 && s.matchAny(s.excludedFiles, file)
385400
}
386401

402+
func (s *fileScanner) isFileIncluded(file string) bool {
403+
if len(s.includedFiles) == 0 {
404+
return true
405+
}
406+
return s.matchAny(s.includedFiles, file)
407+
}
408+
387409
// matchAny checks if the text matches any of the regular expressions
388410
func (s *fileScanner) matchAny(matchers []match.Matcher, text string) bool {
389411
for _, m := range matchers {

‎filebeat/input/filestream/fswatch_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ func TestFileScanner(t *testing.T) {
5252
testCases := map[string]struct {
5353
paths []string
5454
excludedFiles []match.Matcher
55+
includedFiles []match.Matcher
5556
symlinks bool
5657
expectedFiles []string
5758
}{
@@ -66,6 +67,13 @@ func TestFileScanner(t *testing.T) {
6667
},
6768
expectedFiles: []string{includedFilePath},
6869
},
70+
"only include included_files": {
71+
paths: []string{excludedFilePath, includedFilePath},
72+
includedFiles: []match.Matcher{
73+
match.MustCompile(includedFileName),
74+
},
75+
expectedFiles: []string{includedFilePath},
76+
},
6977
"skip directories": {
7078
paths: []string{filepath.Join(tmpDir, directoryPath)},
7179
expectedFiles: []string{},
@@ -78,6 +86,7 @@ func TestFileScanner(t *testing.T) {
7886
t.Run(name, func(t *testing.T) {
7987
cfg := fileScannerConfig{
8088
ExcludedFiles: test.excludedFiles,
89+
IncludedFiles: test.includedFiles,
8190
Symlinks: test.symlinks,
8291
RecursiveGlob: false,
8392
}

‎filebeat/input/filestream/fswatch_test_non_windows.go‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"io/ioutil"
2525
"os"
2626
"path/filepath"
27+
"strconv"
2728
"testing"
2829

2930
"github.com/stretchr/testify/assert"
@@ -44,13 +45,14 @@ func TestFileScannerSymlinks(t *testing.T) {
4445
testCases := map[string]struct {
4546
paths []string
4647
excludedFiles []match.Matcher
48+
includedFiles []match.Matcher
4749
symlinks bool
4850
expectedFiles []string
4951
}{
5052
// covers test_input.py/test_skip_symlinks
5153
"skip symlinks": {
5254
paths: []string{
53-
filepath.Join(tmpDir, "symlink_to_included_file"),
55+
filepath.Join(tmpDir, "symlink_to_0"),
5456
filepath.Join(tmpDir, "included_file"),
5557
},
5658
symlinks: false,
@@ -60,22 +62,37 @@ func TestFileScannerSymlinks(t *testing.T) {
6062
},
6163
"return a file once if symlinks are enabled": {
6264
paths: []string{
63-
filepath.Join(tmpDir, "symlink_to_included_file"),
65+
filepath.Join(tmpDir, "symlink_to_0"),
6466
filepath.Join(tmpDir, "included_file"),
6567
},
6668
symlinks: true,
6769
expectedFiles: []string{
6870
mustAbsPath(filepath.Join(tmpDir, "included_file")),
6971
},
7072
},
73+
"do not return symlink if original file is not allowed": {
74+
paths: []string{
75+
filepath.Join(tmpDir, "symlink_to_1"),
76+
filepath.Join(tmpDir, "included_file"),
77+
},
78+
excludedFiles: []match.Matcher{
79+
match.MustCompile("original_" + excludedFileName),
80+
},
81+
symlinks: true,
82+
expectedFiles: []string{
83+
mustAbsPath(filepath.Join(tmpDir, "included_file")),
84+
},
85+
},
7186
}
7287

73-
err := os.Symlink(
74-
mustAbsPath(filepath.Join(tmpDir, "included_file")),
75-
mustAbsPath(filepath.Join(tmpDir, "symlink_to_included_file")),
76-
)
77-
if err != nil {
78-
t.Fatal(err)
88+
for i, filename := range []string{"included_file", "excluded_file"} {
89+
err := os.Symlink(
90+
mustAbsPath(filepath.Join(tmpDir, "original_"+filename)),
91+
mustAbsPath(filepath.Join(tmpDir, "symlink_to_"+strconv.Itoa(i))),
92+
)
93+
if err != nil {
94+
t.Fatal(err)
95+
}
7996
}
8097

8198
for name, test := range testCases {
@@ -84,6 +101,7 @@ func TestFileScannerSymlinks(t *testing.T) {
84101
t.Run(name, func(t *testing.T) {
85102
cfg := fileScannerConfig{
86103
ExcludedFiles: test.excludedFiles,
104+
IncludedFiles: test.includedFiles,
87105
Symlinks: true,
88106
RecursiveGlob: false,
89107
}
@@ -150,3 +168,11 @@ func TestFileWatcherRenamedFile(t *testing.T) {
150168
assert.Equal(t, testPath, evt.OldPath)
151169
assert.Equal(t, renamedPath, evt.NewPath)
152170
}
171+
172+
func mustAbsPath(filename string) string {
173+
abspath, err := filepath.Abs(filename)
174+
if err != nil {
175+
panic(err)
176+
}
177+
return abspath
178+
}

‎filebeat/input/filestream/input_integration_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,8 @@ func TestFilestreamSymlinkRemoved(t *testing.T) {
860860

861861
// test_truncate from test_harvester.py
862862
func TestFilestreamTruncate(t *testing.T) {
863+
t.Skip("Flaky test: https://github.com/elastic/beats/issues/25217")
864+
863865
env := newInputTestingEnvironment(t)
864866

865867
testlogName := "test.log"

‎generator/_templates/beat/{beat}/tools/tools.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
package tools
66

77
import (
8+
_ "github.com/magefile/mage"
89
_ "github.com/pierrre/gotestcover"
910
_ "github.com/tsg/go-daemon"
1011
_ "golang.org/x/tools/cmd/goimports"
12+
_ "gotest.tools/gotestsum/cmd"
1113

1214
_ "github.com/mitchellh/gox"
1315
_ "golang.org/x/lint/golint"

‎generator/_templates/metricbeat/{beat}/tools/tools.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
package tools
66

77
import (
8+
_ "github.com/magefile/mage"
89
_ "github.com/pierrre/gotestcover"
910
_ "github.com/tsg/go-daemon"
1011
_ "golang.org/x/tools/cmd/goimports"
12+
_ "gotest.tools/gotestsum/cmd"
1113

1214
_ "github.com/mitchellh/gox"
1315
_ "golang.org/x/lint/golint"

0 commit comments

Comments
 (0)