Skip to content

Commit 8786d05

Browse files
blakerouseblakerouse
authored
Update metricbeat to use mage for all scenarios along with Makefile shim (#17799)
* Update metricbeat to use mage and the mage make targets. * Run update and fmt. * Remove assets.go as its no longer used. * Split apart the metricbeat tests in travis into multiple jobs to take less time and not run over the timeline. * Add back in create-metricset with mage. * Re-add test_xpack_base.py. * Add kafka to requirements.txt. * Fix kafka to work with python 3.7. * Remove the extra kafka-python from rebase on master. * Fix issues from code review. * Add back in comment from merge. * Fix comment spacing. * Fix update target in generate metricbeat. * Update magefile in metricbeat generator. * Run fmt and update.
1 parent 29ecd72 commit 8786d05

62 files changed

Lines changed: 252 additions & 304 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.travis.yml‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -111,23 +111,22 @@ jobs:
111111
# Metricbeat
112112
- os: linux
113113
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
114-
env: TARGETS="-C metricbeat unit-tests coverage-report"
114+
env: TARGETS="-C metricbeat unit-tests"
115115
go: $TRAVIS_GO_VERSION
116116
stage: test
117117
- os: linux
118-
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
119-
env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
118+
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
119+
env: TARGETS="-C metricbeat integration-tests"
120120
go: $TRAVIS_GO_VERSION
121121
stage: test
122122
- os: linux
123-
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
124-
env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
123+
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
124+
env: TARGETS="-C metricbeat system-tests"
125125
go: $TRAVIS_GO_VERSION
126126
stage: test
127-
128127
- os: osx
129128
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
130-
env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
129+
env: TARGETS="-C metricbeat testsuite"
131130
go: $TRAVIS_GO_VERSION
132131
stage: test
133132
- os: linux
@@ -137,6 +136,21 @@ jobs:
137136
stage: test
138137
- os: linux
139138
before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
139+
env: TARGETS="-C x-pack/metricbeat unit-tests"
140+
go: $TRAVIS_GO_VERSION
141+
stage: test
142+
- os: linux
143+
before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
144+
env: TARGETS="-C x-pack/metricbeat integration-tests"
145+
go: $TRAVIS_GO_VERSION
146+
stage: test
147+
- os: linux
148+
before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
149+
env: TARGETS="-C x-pack/metricbeat system-tests"
150+
go: $TRAVIS_GO_VERSION
151+
stage: test
152+
- os: osx
153+
before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
140154
env: TARGETS="-C x-pack/metricbeat testsuite"
141155
go: $TRAVIS_GO_VERSION
142156
stage: test

‎Jenkinsfile‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,20 @@ pipeline {
319319
}
320320
}
321321
steps {
322-
mageTarget("Metricbeat OSS linux/amd64 (integTest)", "metricbeat", "integTest")
322+
mageTarget("Metricbeat OSS linux/amd64 (goIntegTest)", "metricbeat", "goIntegTest")
323+
}
324+
}
325+
stage('Metricbeat Python integration tests'){
326+
agent { label 'ubuntu && immutable' }
327+
options { skipDefaultCheckout() }
328+
when {
329+
beforeAgent true
330+
expression {
331+
return env.BUILD_METRICBEAT != "false"
332+
}
333+
}
334+
steps {
335+
mageTarget("Metricbeat OSS linux/amd64 (pythonIntegTest)", "metricbeat", "pythonIntegTest")
323336
}
324337
}
325338
stage('Metricbeat x-pack'){

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ PROJECTS_XPACK_MAGE=$(PROJECTS_XPACK_PKG) x-pack/libbeat
2828
#
2929
# Includes
3030
#
31-
include dev-tools/make/mage.mk
31+
include dev-tools/make/mage-install.mk
3232

3333
## help : Show this help.
3434
help: Makefile

‎dev-tools/make/mage-install.mk‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
MAGE_VERSION ?= v1.9.0
2+
MAGE_PRESENT := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
3+
MAGE_IMPORT_PATH ?= github.com/magefile/mage
4+
export MAGE_IMPORT_PATH
5+
6+
.PHONY: mage
7+
mage:
8+
ifndef MAGE_PRESENT
9+
@echo Installing mage $(MAGE_VERSION) from vendor dir.
10+
@go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH}/...
11+
@-mage -clean
12+
endif
13+
@true

‎dev-tools/make/mage.mk‎

Lines changed: 69 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,69 @@
1-
MAGE_VERSION ?= v1.9.0
2-
MAGE_PRESENT := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
3-
MAGE_IMPORT_PATH ?= github.com/magefile/mage
4-
export MAGE_IMPORT_PATH
5-
6-
.PHONY: mage
7-
mage:
8-
ifndef MAGE_PRESENT
9-
@echo Installing mage $(MAGE_VERSION) from vendor dir.
10-
@go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH}/...
11-
@-mage -clean
12-
endif
13-
@true
1+
# This is a minimal Makefile for Beats that are built with Mage. Its only
2+
# responsibility is to provide compatibility with existing Jenkins and Travis
3+
# setups.
4+
5+
#
6+
# Variables
7+
#
8+
.DEFAULT_GOAL := help
9+
PWD := $(CURDIR)
10+
11+
#
12+
# Includes
13+
#
14+
include $(ES_BEATS)/dev-tools/make/mage-install.mk
15+
16+
#
17+
# Targets (alphabetically sorted).
18+
#
19+
.PHONY: check
20+
check: mage
21+
mage check
22+
23+
.PHONY: clean
24+
clean: mage
25+
mage clean
26+
27+
fix-permissions:
28+
29+
.PHONY: fmt
30+
fmt: mage
31+
mage fmt
32+
33+
# Default target.
34+
.PHONY: help
35+
help:
36+
@echo Use mage rather than make. Here are the available mage targets:
37+
@mage -l
38+
39+
.PHONY: release
40+
release: mage
41+
mage package
42+
43+
stop-environment:
44+
45+
.PHONY: unit-tests
46+
unit-tests: mage
47+
mage unitTest
48+
49+
.PHONY: integration-tests
50+
integration-tests: mage
51+
rm -f build/TEST-go-integration.out
52+
mage goIntegTest || ( cat build/TEST-go-integration.out && false )
53+
54+
.PHONY: system-tests
55+
system-tests: mage
56+
mage pythonIntegTest
57+
58+
.PHONY: testsuite
59+
testsuite: mage
60+
rm -f build/TEST-go-integration.out
61+
mage update build unitTest integTest || ( cat build/TEST-go-integration.out && false )
62+
63+
.PHONY: update
64+
update: mage
65+
mage update
66+
67+
.PHONY: crosscompile
68+
crosscompile: mage
69+
mage crossBuild

‎dev-tools/make/xpack.mk‎

Lines changed: 0 additions & 53 deletions
This file was deleted.

‎filebeat/docs/fields.asciidoc‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28240,6 +28240,15 @@ type: keyword
2824028240
Palo Alto Networks name for the threat.
2824128241
2824228242
28243+
type: keyword
28244+
28245+
--
28246+
28247+
*`panw.panos.action`*::
28248+
+
28249+
--
28250+
Action taken for the session.
28251+
2824328252
type: keyword
2824428253
2824528254
--

‎generator/_templates/metricbeat/{beat}/magefile.go‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"github.com/elastic/beats/v7/dev-tools/mage/target/common"
1515
"github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
1616
"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
17-
"github.com/elastic/beats/v7/dev-tools/mage/target/update"
1817
"github.com/elastic/beats/v7/generator/common/beatgen"
1918
metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage"
2019
)
@@ -45,7 +44,7 @@ func Package() {
4544

4645
devtools.UseCommunityBeatPackaging()
4746

48-
mg.Deps(update.Update)
47+
mg.Deps(Update)
4948
mg.Deps(build.CrossBuild, build.CrossBuildGoDaemon)
5049
mg.SerialDeps(devtools.Package, pkg.PackageTest)
5150
}
@@ -99,9 +98,14 @@ func Fmt() {
9998
common.Fmt()
10099
}
101100

102-
// Update updates the generated files (aka make update).
103-
func Update() error {
104-
return update.Update()
101+
// Update is an alias for running fields, dashboards, config.
102+
func Update() {
103+
mg.SerialDeps(Fields, Dashboards, Config, Imports)
104+
}
105+
106+
// Dashboards collects all the dashboards and generates index patterns.
107+
func Dashboards() error {
108+
return devtools.KibanaDashboards("module")
105109
}
106110

107111
// Imports generates an include/list.go file containing

‎generator/common/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ BEAT_PATH=${GOPATH}/src/${BEAT_NAME}
55
ES_BEATS=${GOPATH}/src/github.com/elastic/beats
66
PREPARE_COMMAND?=
77

8-
-include ${ES_BEATS}/dev-tools/make/mage.mk
8+
-include ${ES_BEATS}/dev-tools/make/mage-install.mk
99

1010
# Runs test build for mock beat
1111
.PHONY: test

‎libbeat/publisher/pipeline/testing.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,14 @@ func (b *mockBatch) Events() []publisher.Event {
9696
return b.events
9797
}
9898

99-
func (b *mockBatch) ACK() { signalFn(b.onACK) }
100-
func (b *mockBatch) Drop() { signalFn(b.onDrop) }
101-
func (b *mockBatch) Retry() { signalFn(b.onRetry) }
102-
func (b *mockBatch) Cancelled() { signalFn(b.onCancelled) }
103-
func (b *mockBatch) RetryEvents(events []publisher.Event) { b.updateEvents(events); signalFn(b.onRetry) }
99+
func (b *mockBatch) ACK() { signalFn(b.onACK) }
100+
func (b *mockBatch) Drop() { signalFn(b.onDrop) }
101+
func (b *mockBatch) Retry() { signalFn(b.onRetry) }
102+
func (b *mockBatch) Cancelled() { signalFn(b.onCancelled) }
103+
func (b *mockBatch) RetryEvents(events []publisher.Event) {
104+
b.updateEvents(events)
105+
signalFn(b.onRetry)
106+
}
104107

105108
func (b *mockBatch) reduceTTL() bool {
106109
if b.onReduceTTL != nil {

0 commit comments

Comments
 (0)