Skip to content

Commit 68bf3fc

Browse files
jsorianojsoriano
committed
Merge remote-tracking branch 'origin/7.x' into backport_18125_7.x
2 parents 9407ab5 + 76cb884 commit 68bf3fc

510 files changed

Lines changed: 23233 additions & 5715 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.ci/scripts/install-terraform.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/usr/bin/env bash
2+
3+
set -exuo pipefail
4+
5+
MSG="parameter missing."
6+
TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG}
7+
HOME=${HOME:?$MSG}
8+
TERRAFORM_CMD="${HOME}/bin/terraform"
9+
10+
OS=$(uname -s | tr '[:upper:]' '[:lower:]')
11+
12+
mkdir -p "${HOME}/bin"
13+
14+
curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip
15+
unzip -o ${TERRAFORM_CMD}.zip -d $(dirname ${TERRAFORM_CMD})
16+
rm ${TERRAFORM_CMD}.zip
17+
18+
chmod +x "${TERRAFORM_CMD}"

‎.ci/scripts/terraform-cleanup.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
set -exuo pipefail
4+
5+
DIRECTORY=${1:-.}
6+
7+
FAILED=0
8+
for tfstate in $(find $DIRECTORY -name terraform.tfstate); do
9+
cd $(dirname $tfstate)
10+
if ! terraform destroy -auto-approve; then
11+
FAILED=1
12+
fi
13+
cd -
14+
done
15+
16+
exit $FAILED

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,7 @@ x-pack/dockerlogbeat/temproot.tar
3939
*.test
4040
*.prof
4141
*.pyc
42+
43+
# Terraform
44+
*.terraform
45+
*.tfstate*

‎CHANGELOG.next.asciidoc‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
3333
- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
3434
- CEF extensions are now mapped to the data types defined in the CEF guide. {pull}14342[14342]
3535
- Improve ECS field mappings in panw module. event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
36+
- Improve ECS categorization field mappings for nginx module. http.request.referrer is now lowercase & http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
3637

3738
*Heartbeat*
3839

@@ -266,6 +267,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
266267
- Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
267268
- Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464]
268269
- Add Kerberos support to Elasticsearch output. {pull}17927[17927]
270+
- Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
271+
- Add keystore support for autodiscover static configurations. {pull]16306[16306]
272+
- Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153]
269273

270274
*Auditbeat*
271275

@@ -351,6 +355,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
351355
- Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
352356
- Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
353357
- Improve ECS categorization field mappings for nginx module. {issue}16174[16174] {pull}17844[17844]
358+
- Add support for Google Application Default Credentials to the Google Pub/Sub input and Google Cloud modules. {pull}15668[15668]
359+
- Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
360+
- Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108]
361+
- Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065]
354362

355363
*Heartbeat*
356364

@@ -428,6 +436,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
428436
- Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429]
429437
- Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725]
430438
- Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609]
439+
- Add Metricbeat IIS module dashboards. {pull}17966[17966]
431440
- Add dashboard for the azure database account metricset. {pull}17901[17901]
432441
- Allow partial region and zone name in googlecloud module config. {pull}17913[17913]
433442
- Add aggregation aligner as a config parameter for googlecloud stackdriver metricset. {issue}17141[[17141] {pull}17719[17719]
@@ -448,6 +457,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
448457
- Add more DNS error codes to the Sysmon module. {issue}15685[15685]
449458
- Add Audit and Log Management, Computer Object Management, and Distribution Group related events to the Security module. {pull}15217[15217]
450459
- Add experimental event log reader implementation that should be faster in most cases. {issue}6585[6585] {pull}16849[16849]
460+
- Set process.command_line and process.parent.command_line from Sysmon Event ID 1. {pull}17327[17327]
461+
- Add support for event IDs 4673,4674,4697,4698,4699,4700,4701,4702,4768,4769,4770,4771,4776,4778,4779,4964 to the Security module {pull}17517[17517]
451462

452463
==== Deprecated
453464

‎Jenkinsfile‎

Lines changed: 126 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ pipeline {
1515
BASE_DIR = 'src/github.com/elastic/beats'
1616
GOX_FLAGS = "-arch amd64"
1717
DOCKER_COMPOSE_VERSION = "1.21.0"
18+
TERRAFORM_VERSION = "0.12.24"
1819
PIPELINE_LOG_LEVEL = "INFO"
1920
DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
2021
DOCKER_REGISTRY = 'docker.elastic.co'
22+
AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
2123
RUNBLD_DISABLE_NOTIFICATIONS = 'true'
2224
}
2325
options {
@@ -36,6 +38,11 @@ pipeline {
3638
booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
3739
booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
3840
booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
41+
42+
booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.')
43+
booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.')
44+
string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
45+
3946
booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
4047
booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
4148
}
@@ -352,8 +359,30 @@ pipeline {
352359
return env.BUILD_METRICBEAT_XPACK != "false"
353360
}
354361
}
355-
steps {
356-
mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
362+
stages {
363+
stage('Prepare cloud integration tests environments'){
364+
agent { label 'ubuntu && immutable' }
365+
options { skipDefaultCheckout() }
366+
steps {
367+
startCloudTestEnv('x-pack-metricbeat', [
368+
[cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
369+
])
370+
}
371+
}
372+
stage('Metricbeat x-pack'){
373+
agent { label 'ubuntu && immutable' }
374+
options { skipDefaultCheckout() }
375+
steps {
376+
withCloudTestEnv() {
377+
mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "build test")
378+
}
379+
}
380+
}
381+
}
382+
post {
383+
cleanup {
384+
terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
385+
}
357386
}
358387
}
359388
stage('Metricbeat crosscompile'){
@@ -671,7 +700,7 @@ def withBeatsEnv(boolean archive, Closure body) {
671700
"TEST_COVERAGE=true",
672701
"RACE_DETECTOR=true",
673702
"PYTHON_ENV=${WORKSPACE}/python-env",
674-
"TEST_TAGS=oracle",
703+
"TEST_TAGS=${env.TEST_TAGS},oracle",
675704
"DOCKER_PULL=0",
676705
]) {
677706
deleteDir()
@@ -738,6 +767,7 @@ def installTools() {
738767
if(isUnix()) {
739768
retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") }
740769
retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") }
770+
retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
741771
retry(i) { sh(label: "Install Mage", script: "make mage") }
742772
} else {
743773
retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
@@ -809,6 +839,7 @@ def dumpFilteredEnvironment(){
809839
echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
810840
echo "STRESS_TESTS: ${env.STRESS_TESTS}"
811841
echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}"
842+
echo "TEST_TAGS: ${env.TEST_TAGS}"
812843
echo "GOX_OS: ${env.GOX_OS}"
813844
echo "GOX_OSARCH: ${env.GOX_OSARCH}"
814845
echo "GOX_FLAGS: ${env.GOX_FLAGS}"
@@ -895,6 +926,98 @@ def isChangedXPackCode(patterns) {
895926
return isChanged(allPatterns)
896927
}
897928

929+
// withCloudTestEnv executes a closure with credentials for cloud test
930+
// environments.
931+
def withCloudTestEnv(Closure body) {
932+
def maskedVars = []
933+
def testTags = "${env.TEST_TAGS}"
934+
935+
// AWS
936+
if (params.allCloudTests || params.awsCloudTests) {
937+
testTags = "${testTags},aws"
938+
def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
939+
if (!aws.containsKey('access_key')) {
940+
error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'")
941+
}
942+
if (!aws.containsKey('secret_key')) {
943+
error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'")
944+
}
945+
maskedVars.addAll([
946+
[var: "AWS_REGION", password: params.awsRegion],
947+
[var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
948+
[var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
949+
])
950+
}
951+
952+
withEnv([
953+
"TEST_TAGS=${testTags}",
954+
]) {
955+
withEnvMask(vars: maskedVars) {
956+
body()
957+
}
958+
}
959+
}
960+
961+
def terraformInit(String directory) {
962+
dir(directory) {
963+
sh(label: "Terraform Init on ${directory}", script: "terraform init")
964+
}
965+
}
966+
967+
def terraformApply(String directory) {
968+
terraformInit(directory)
969+
dir(directory) {
970+
sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
971+
}
972+
}
973+
974+
// Start testing environment on cloud using terraform. Terraform files are
975+
// stashed so they can be used by other stages. They are also archived in
976+
// case manual cleanup is needed.
977+
//
978+
// Example:
979+
// startCloudTestEnv('x-pack-metricbeat', [
980+
// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'],
981+
// ])
982+
// ...
983+
// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat')
984+
def startCloudTestEnv(String name, environments = []) {
985+
withCloudTestEnv() {
986+
withBeatsEnv(false) {
987+
def runAll = params.runAllCloudTests
988+
try {
989+
for (environment in environments) {
990+
if (environment.cond || runAll) {
991+
retry(2) {
992+
terraformApply(environment.dir)
993+
}
994+
}
995+
}
996+
} finally {
997+
// Archive terraform states in case manual cleanup is needed.
998+
archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
999+
}
1000+
stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
1001+
}
1002+
}
1003+
}
1004+
1005+
1006+
// Looks for all terraform states in directory and runs terraform destroy for them,
1007+
// it uses terraform states previously stashed by startCloudTestEnv.
1008+
def terraformCleanup(String stashName, String directory) {
1009+
stage("Remove cloud scenarios in ${directory}"){
1010+
withCloudTestEnv() {
1011+
withBeatsEnv(false) {
1012+
unstash "terraform-${stashName}"
1013+
retry(2) {
1014+
sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
1015+
}
1016+
}
1017+
}
1018+
}
1019+
}
1020+
8981021
def loadConfigEnvVars(){
8991022
def empty = []
9001023
env.GO_VERSION = readFile(".go-version").trim()

‎auditbeat/docs/fields.asciidoc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2458,7 +2458,8 @@ Contains common beat fields available in all event types.
24582458
*`agent.hostname`*::
24592459
+
24602460
--
2461-
Hostname of the agent.
2461+
Deprecated - use agent.name or agent.id to identify an agent. Hostname of the agent.
2462+
24622463
24632464
type: keyword
24642465

‎auditbeat/include/fields.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dev-tools/mage/common.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -833,3 +833,28 @@ func ListMatchingEnvVars(prefixes ...string) []string {
833833
}
834834
return vars
835835
}
836+
837+
// IntegrationTestEnvVars returns the names of environment variables needed to configure
838+
// connections to integration test environments.
839+
func IntegrationTestEnvVars() []string {
840+
// Environment variables that can be configured with paths to files
841+
// with authentication information.
842+
vars := []string{
843+
"AWS_SHARED_CREDENTIAL_FILE",
844+
"AZURE_AUTH_LOCATION",
845+
"GOOGLE_APPLICATION_CREDENTIALS",
846+
}
847+
// Environment variables with authentication information.
848+
prefixes := []string{
849+
"AWS_",
850+
"AZURE_",
851+
852+
// Accepted by terraform, but not by many clients, including Beats
853+
"GOOGLE_",
854+
"GCLOUD_",
855+
}
856+
for _, prefix := range prefixes {
857+
vars = append(vars, ListMatchingEnvVars(prefix)...)
858+
}
859+
return vars
860+
}

‎dev-tools/mage/config.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error
116116
"UseDockerMetadataProcessor": true,
117117
"UseKubernetesMetadataProcessor": false,
118118
"ExcludeDashboards": false,
119+
"UseProcessorsTemplate": false,
119120
}
120121
for k, v := range args.ExtraVars {
121122
params[k] = v

‎dev-tools/mage/gotest.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,9 @@ func GoTestIntegrationForModule(ctx context.Context) error {
156156
foundModule = true
157157

158158
// Set MODULE because only want that modules tests to run inside the testing environment.
159-
runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), map[string]string{"MODULE": fi.Name()})
159+
env := map[string]string{"MODULE": fi.Name()}
160+
passThroughEnvs(env, IntegrationTestEnvVars()...)
161+
runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env)
160162
if err != nil {
161163
return errors.Wrapf(err, "test setup failed for module %s", fi.Name())
162164
}

0 commit comments

Comments
 (0)