Skip to content

Commit 5bc2d55

Browse files
author
kaiyan-sheng
committed
Merge remote-tracking branch 'upstream/master' into s3_test
2 parents 1670a9b + 35e2e99 commit 5bc2d55

35 files changed

Lines changed: 637 additions & 96 deletions

‎.ci/packaging.groovy‎

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ pipeline {
4040
parameters {
4141
booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
4242
booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
43+
booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
4344
}
4445
stages {
4546
stage('Filter build') {
@@ -83,12 +84,13 @@ pipeline {
8384
}
8485
}
8586
setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
87+
// Stash without any build/dependencies context to support different architectures.
88+
stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
8689
withMageEnv(){
8790
dir("${BASE_DIR}"){
8891
setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
8992
}
9093
}
91-
stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
9294
}
9395
}
9496
stage('Build Packages'){
@@ -172,12 +174,73 @@ pipeline {
172174
}
173175
steps {
174176
withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") {
175-
deleteDir()
177+
deleteWorkspace()
176178
withMacOSEnv(){
177179
release()
178180
}
179181
}
180182
}
183+
post {
184+
always {
185+
// static workers require this
186+
deleteWorkspace()
187+
}
188+
}
189+
}
190+
}
191+
}
192+
}
193+
stage('Build Packages ARM'){
194+
matrix {
195+
axes {
196+
axis {
197+
name 'BEATS_FOLDER'
198+
values (
199+
'auditbeat',
200+
'filebeat',
201+
'heartbeat',
202+
'journalbeat',
203+
'metricbeat',
204+
'packetbeat',
205+
'x-pack/auditbeat',
206+
'x-pack/elastic-agent',
207+
'x-pack/filebeat',
208+
'x-pack/heartbeat',
209+
'x-pack/metricbeat',
210+
'x-pack/packetbeat'
211+
)
212+
}
213+
}
214+
stages {
215+
stage('Package Docker images for linux/arm64'){
216+
agent { label 'arm' }
217+
options { skipDefaultCheckout() }
218+
when {
219+
beforeAgent true
220+
expression {
221+
return params.arm
222+
}
223+
}
224+
environment {
225+
HOME = "${env.WORKSPACE}"
226+
PACKAGES = "docker"
227+
PLATFORMS = [
228+
'linux/arm64',
229+
].join(' ')
230+
}
231+
steps {
232+
withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
233+
deleteWorkspace()
234+
release()
235+
pushCIDockerImages()
236+
}
237+
}
238+
post {
239+
always {
240+
// static workers require this
241+
deleteWorkspace()
242+
}
243+
}
181244
}
182245
}
183246
}
@@ -408,14 +471,43 @@ def getBeatsName(baseDir) {
408471
}
409472

410473
def withBeatsEnv(Closure body) {
474+
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
475+
fixPermissions()
411476
withMageEnv(){
412477
withEnv([
413478
"PYTHON_ENV=${WORKSPACE}/python-env"
414479
]) {
415-
unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
416480
dir("${env.BASE_DIR}"){
417481
body()
418482
}
419483
}
420484
}
421485
}
486+
487+
/**
488+
* This method fixes the filesystem permissions after the build has happenend. The reason is to
489+
* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental
490+
* issues.
491+
*/
492+
def deleteWorkspace() {
493+
catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
494+
fixPermissions()
495+
deleteDir()
496+
}
497+
}
498+
499+
def fixPermissions() {
500+
if(isUnix()) {
501+
catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
502+
dir("${env.BASE_DIR}") {
503+
if (fileExists('script/fix_permissions.sh')) {
504+
sh(label: 'Fix permissions', script: """#!/usr/bin/env bash
505+
set +x
506+
source ./dev-tools/common.bash
507+
docker_setup
508+
script/fix_permissions.sh ${WORKSPACE}""", returnStatus: true)
509+
}
510+
}
511+
}
512+
}
513+
}

‎CHANGELOG.asciidoc‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,233 @@
33
:issue: https://github.com/elastic/beats/issues/
44
:pull: https://github.com/elastic/beats/pull/
55

6+
[[release-notes-7.11.0]]
7+
=== Beats version 7.11.0
8+
https://github.com/elastic/beats/compare/v7.10.2...v7.11.0[View commits]
9+
10+
==== Breaking changes
11+
12+
*Affecting all Beats*
13+
14+
- Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs. {pull}21179[21179]
15+
- Update to ECS 1.7.0. {pull}22571[22571]
16+
- Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
17+
18+
*Auditbeat*
19+
20+
- Use ECS 1.7 ingress/egress network directions instead of inbound/outbound for system/socket. {pull}22991[22991]
21+
- Use ingress/egress instead of inbound/outbound for ECS 1.7 in auditd module. {pull}23000[23000]
22+
23+
*Filebeat*
24+
25+
- Add fileset to ingest Kibana's ECS audit logs. {pull}22696[22696]
26+
- Remove `suricata.eve.timestamp` alias field. {issue}10535[10535] {pull}22095[22095]
27+
- Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571]
28+
- Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
29+
- Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
30+
31+
*Heartbeat*
32+
- Adds negative body match. {pull}20728[20728]
33+
34+
*Metricbeat*
35+
36+
- Change cloud.provider from googlecloud to gcp. {pull}21775[21775]
37+
- Rename googlecloud module to gcp module. {pull}22246[22246]
38+
- Use ingress/egress instead of inbound/outbound for system/socket metricset. {pull}22992[22992]
39+
- Change types of numeric metrics from Kubelet summary api to double so as to cover big numbers. {pull}23335[23335]
40+
41+
*Packetbeat*
42+
43+
- Update how Packetbeat classifies network directionality to bring it in line with ECS 1.7 {pull}22996[22996]
44+
45+
*Winlogbeat*
46+
47+
- Use ECS 1.7 ingress/egress instead of inbound/outbound network.direction in sysmon. {pull}22997[22997]
48+
49+
==== Bugfixes
50+
51+
*Affecting all Beats*
52+
53+
- Fix memory leak and events duplication in docker autodiscover and add_docker_metadata. {pull}21851[21851]
54+
- Fix duplicated pod events in kubernetes autodiscover for pods with init or ephemeral containers. {pull}22438[22438]
55+
- Fix FileVersion contained in Windows exe files. {pull}22581[22581]
56+
- Log debug message if the Kibana dashboard can not be imported from the archive because of the invalid archive directory structure {issue}12211[12211], {pull}13387[13387]
57+
- Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss` as gauges (rather than counters). {pull}22877[22877]
58+
- Use PROGRAMDATA environment variable instead of C:\ProgramData for windows install service {pull}22874[22874]
59+
- Fix reporting of cgroup metrics when running under Docker {pull}22879[22879]
60+
- Fix typo in config docs {pull}23185[23185]
61+
- Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
62+
- Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
63+
64+
*Auditbeat*
65+
66+
- file_integrity: stop monitoring excluded paths {issue}21278[21278] {pull}21282[21282]
67+
- Note incompatibility of system/socket on ARM. {pull}23381[23381]
68+
69+
*Filebeat*
70+
71+
- Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
72+
- Fix network.direction logic in zeek connection fileset. {pull}22967[22967]
73+
- Fix aws s3 overview dashboard. {pull}23045[23045]
74+
- Fix bad `network.direction` values in Fortinet/firewall fileset. {pull}23072[23072]
75+
- Fix Cisco ASA/FTD module's parsing of WebVPN log message 716002. {pull}22966[22966]
76+
- Add support for organization and custom prefix in AWS/CloudTrail fileset. {issue}23109[23109] {pull}23126[23126]
77+
- Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204]
78+
- Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273]
79+
- Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534]
80+
- Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777]
81+
82+
*Heartbeat*
83+
84+
- Fixed missing `tls` fields when connecting to https via proxy. {issue}15797[15797] {pull}22190[22190]
85+
86+
*Metricbeat*
87+
88+
- Change Session ID type from int to string {pull}22359[22359]
89+
- Fix filesystem types on Windows in filesystem metricset. {pull}22531[22531]
90+
- Fix failiures caused by custom beat names with more than 15 characters {pull}22550[22550]
91+
- Update NATS dashboards to leverage connection and route metricsets {pull}22646[22646]
92+
- Fix rate metrics in Kafka broker metricset by using last minute rate instead of mean rate. {pull}22733[22733]
93+
- Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
94+
- Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
95+
96+
*Packetbeat*
97+
98+
- Fix SIP parser logic related to line length check. {pull}23411[23411]
99+
100+
101+
*Winlogbeat*
102+
103+
- Protect against accessing an undefined variable in Security module. {pull}22937[22937]
104+
- Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
105+
106+
==== Added
107+
108+
*Affecting all Beats*
109+
110+
- Add istiod metricset. {pull}21519[21519]
111+
- Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590]
112+
- Add cloud.account.id for GCP into add_cloud_metadata processor. {pull}21776[21776]
113+
- Add proxy metricset for istio module. {pull}21751[21751]
114+
- Add kubernetes.node.hostname metadata of Kubernetes node. {pull}22189[22189]
115+
- Enable always add_resource_metadata for Pods and Services of kubernetes autodiscovery. {pull}22189[22189]
116+
- Add add_resource_metadata option setting (always enabled) for add_kubernetes_metadata setting. {pull}22189[22189]
117+
- Add support for ephemeral containers in kubernetes autodiscover and `add_kubernetes_metadata`. {pull}22389[22389] {pull}22439[22439]
118+
- Added support for wildcard fields and keyword fallback in beats setup commands. {pull}22521[22521]
119+
- Fix polling node when it is not ready and monitor by hostname {pull}22666[22666]
120+
- Add `expand_keys` option to `decode_json_fields` processor and `json` input, to recusively de-dot and expand json keys into hierarchical object structures {pull}22849[22849]
121+
- Update k8s client and release k8s leader lock gracefully {pull}22919[22919]
122+
- Improve event normalization performance {pull}22974[22974]
123+
- Add tini as init system in docker images {pull}22137[22137]
124+
- Added "detect_mime_type" processor for detecting mime types {pull}22940[22940]
125+
- Added "add_network_direction" processor for determining perimeter-based network direction. {pull}23076[23076]
126+
- Added new `rate_limit` processor for enforcing rate limits on event throughput. {pull}22883[22883]
127+
- Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
128+
- Improve equals check. {pull}22778[22778]
129+
130+
*Auditbeat*
131+
132+
- Add several improvements for auditd module for improved ECS field mapping {pull}22647[22647]
133+
- Add ECS 1.7 `configuration` categorization in certain events in auditd module. {pull}23000[23000]
134+
135+
*Filebeat*
136+
137+
138+
- Adding support for Oracle Database Audit Logs {pull}21991[21991]
139+
- Add max_number_of_messages config into s3 input. {pull}21993[21993]
140+
- Add SSL option to checkpoint module {pull}19560[19560]
141+
- Added support for MySQL Enterprise audit logs. {pull}22273[22273]
142+
- Rename googlecloud module to gcp module. {pull}22214[22214]
143+
- Rename awscloudwatch input to aws-cloudwatch. {pull}22228[22228]
144+
- Rename google-pubsub input to gcp-pubsub. {pull}22213[22213]
145+
- Copy tag names from MISP data into events. {pull}21664[21664]
146+
- Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696]
147+
- Add platform logs in the azure filebeat module. {pull}22371[22371]
148+
- Added `event.ingested` field to data from the Netflow module. {pull}22412[22412]
149+
- Improve panw ECS url fields mapping. {pull}22481[22481]
150+
- Improve Nats filebeat dashboard. {pull}22726[22726]
151+
- Add support for UNIX datagram sockets in `unix` input. {issues}18632[18632] {pull}22699[22699]
152+
- Add `http.request.mime_type` for Elasticsearch audit log fileset. {pull}22975[22975]
153+
- Add new httpjson input features and mark old config ones for deprecation {pull}22320[22320]
154+
- Add configuration option to set external and internal networks for panw panos fileset {pull}22998[22998]
155+
- Add `subbdomain` fields for rsa2elk modules. {pull}23035[23035]
156+
- Add subdomain enrichment for suricata/eve fileset. {pull}23011[23011]
157+
- Add subdomain enrichment for zeek/dns fileset. {pull}23011[23011]
158+
- Add `event.category` "configuration" to auditd module events. {pull}23010[23010]
159+
- Add `event.category` "configuration" to gsuite module events. {pull}23010[23010]
160+
- Add `event.category` "configuration" to o365 module events. {pull}23010[23010]
161+
- Add `event.category` "configuration" to zoom module events. {pull}23010[23010]
162+
- Add `network.direction` to auditd/log fileset. {pull}23041[23041]
163+
- Add logic for external network.direction in sophos xg fileset {pull}22973[22973]
164+
- Preserve AWS CloudTrail eventCategory in aws.cloudtrail.event_category. {issue}22776[22776] {pull}22805[22805]
165+
- Add top_level_domain enrichment for suricata/eve fileset. {pull}23046[23046]
166+
- Add top_level_domain enrichment for zeek/dns fileset. {pull}23046[23046]
167+
- Add `observer.egress.zone` and `observer.ingress.zone` for cisco/asa and cisco/ftd filesets. {pull}23068[23068]
168+
- Allow cisco/asa and cisco/ftd filesets to override network directionality based off of zones. {pull}23068[23068]
169+
- Allow cef and checkpoint modules to override network directionality based off of zones {pull}23066[23066]
170+
- Add `network.direction` to netflow/log fileset. {pull}23052[23052]
171+
- Add the ability to override `network.direction` based on interfaces in Fortinet/firewall fileset. {pull}23072[23072]
172+
- Add `network.direction` override by specifying `internal_networks` in gcp module. {pull}23081[23081]
173+
- Migrate microsoft/defender_atp to httpjson v2 config {pull}23017[23017]
174+
- Migrate microsoft/m365_defender to httpjson v2 config {pull}23018[23018]
175+
- Migrate okta to httpjson v2 config {pull}23059[23059]
176+
- Add support for Snyk Vulnerability and Audit API. {pull}22677[22677]
177+
- Misp improvements: Migration to httpjson v2 config, pagination and deduplication ID {pull}23070[23070]
178+
- Add Google Workspace module and mark Gsuite module as deprecated {pull}22950[22950]
179+
- Mark m365 defender, defender atp, okta and google workspace modules as GA {pull}23113[23113]
180+
- Added `alternative_host` option to google pubsub input {pull}23215[23215]
181+
182+
*Heartbeat*
183+
184+
- Add mime type detection for http responses. {pull}22976[22976]
185+
186+
*Metricbeat*
187+
188+
- Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703]
189+
- Duplicate system.process.cmdline field with process.command_line ECS field name. {pull}22325[22325]
190+
- Add awsfargate module task_stats metricset to monitor AWS ECS Fargate. {pull}22034[22034]
191+
- Add connection and route metricsets for nats metricbeat module to collect metrics per connection/route. {pull}22445[22445]
192+
- Add unit file states to system/service {pull}22557[22557]
193+
- `kibana` module: `stats` metricset no-longer collects usage-related data. {pull}22732[22732]
194+
- Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
195+
- Add io.ops in fields exported by system.diskio. {pull}22066[22066]
196+
- Adjust the Apache status fields in the fleet mode. {pull}22821[22821]
197+
- Add AWS Fargate overview dashboard. {pull}22941[22941]
198+
- Add process.state, process.cpu.pct, process.cpu.start_time and process.memory.pct. {pull}22845[22845]
199+
- Move IIS module to GA and map fields. {issue}22609[22609] {pull}23024[23024]
200+
- Apache: convert status.total_kbytes to status.total_bytes in fleet mode. {pull}23022[23022]
201+
- Release MSSQL as GA {pull}23146[23146]
202+
203+
*Packetbeat*
204+
205+
- Add support for overriding the published index on a per-protocol/flow basis. {pull}22134[22134]
206+
- Change build process for x-pack distribution {pull}21979[21979]
207+
- Tuned the internal queue size to reduce the chances of events being dropped. {pull}22650[22650]
208+
- Add support for "http.request.mime_type" and "http.response.mime_type". {pull}22940[22940]
209+
210+
*Winlogbeat*
211+
212+
- Add file.pe and process.pe fields to ProcessCreate & LoadImage events in Sysmon module. {issue}17335[17335] {pull}22217[22217]
213+
- Add dns.question.subdomain fields for sysmon DNS events. {pull}22999[22999]
214+
- Add additional event categorization for security and sysmon modules. {pull}22988[22988]
215+
- Add dns.question.top_level_domain fields for sysmon DNS events. {pull}23046[23046]
216+
217+
*Elastic Log Driver*
218+
219+
- Add new winlogbeat security dashboard {pull}18775[18775]
220+
221+
==== Deprecated
222+
223+
*Filebeat*
224+
225+
- The experimental modules for Citrix Netscaler and Symantec Endpoint Protection have been removed.
226+
As we continue to expand our coverage of common security data sources, we may consider supporting
227+
Citrix Netscaler and Symantec Endpoint Protection in a future release. {issue}23129[23129] {pull}23130[23130]
228+
229+
==== Known Issue
230+
231+
232+
6233
[[release-notes-7.10.2]]
7234
=== Beats version 7.10.2
8235
https://github.com/elastic/beats/compare/v7.10.1\...v7.10.2[View commits]

0 commit comments

Comments
 (0)