Skip to content

Commit 1bf8087

Browse files
jsorianokaiyan-shengruflin
authored
Migrate docker autodiscovery to ECS (#10898)
Fields injected by docker autodiscover provider were being placed in alias fields introduced for ECS, change them to the new location and add selectors accordingly. This PR includes #10862 and #10758 As a summary: * Autodiscover selectors using ECS structure are added to autodiscover events, old selectors are kept for backwards compatibility * Autodiscover generated metadata follows ECS * Dedotting of labels is added, enabled by default, will be backported for 6.7, but disabled `docker.containers.labels` is not migrated, as it wasn't for `add_docker_metadata` (see #9412) Fixes #10757 Co-Authored-By: kaiyan-sheng <kaiyan.sheng@elastic.co> Co-Authored-By: Nicolas Ruflin <spam@ruflin.com>
1 parent 59a05ed commit 1bf8087

8 files changed

Lines changed: 241 additions & 45 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
168168
- Fixed data types for various hosts fields in `mongodb/replstatus` metricset {pull}10307[10307]
169169
- Added function to close sql database connection. {pull}10355[10355]
170170
- Fix issue with `elasticsearch/node_stats` metricset (x-pack) not indexing `source_node` field. {pull}10639[10639]
171+
- Migrate docker autodiscover to ECS. {issue}10757[10757] {pull}10862[10862]
171172

172173
*Packetbeat*
173174

‎filebeat/tests/system/test_autodiscover.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ def test_docker(self):
5050

5151
# Check metadata is added
5252
assert output[0]['message'] == 'Busybox output 1'
53-
assert output[0]['docker']['container']['image'] == 'busybox'
53+
assert output[0]['container']['image']['name'] == 'busybox'
5454
assert output[0]['docker']['container']['labels'] == {}
55-
assert 'name' in output[0]['docker']['container']
55+
assert 'name' in output[0]['container']
56+
57+
self.assert_fields_are_documented(output[0])

‎heartbeat/tests/system/test_autodiscovery.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ def test_docker(self):
6060
# We don't check all the docker fields because this is really the responsibility
6161
# of libbeat's autodiscovery code.
6262
event = output[0]
63-
if event['monitor']['id'] == 'myid' and event['docker']['container']['id'] is not None:
63+
if event['monitor']['id'] == 'myid' and event['container']['id'] is not None:
6464
matched = True
6565

6666
assert matched
67+
68+
self.assert_fields_are_documented(output[0])

‎libbeat/autodiscover/providers/docker/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,14 @@ type Config struct {
3232
Builders []*common.Config `config:"builders"`
3333
Appenders []*common.Config `config:"appenders"`
3434
Templates template.MapperSettings `config:"templates"`
35+
Dedot bool `config:"labels.dedot"`
3536
}
3637

3738
func defaultConfig() *Config {
3839
return &Config{
3940
Host: "unix:///var/run/docker.sock",
4041
Prefix: "co.elastic",
42+
Dedot: true,
4143
}
4244
}
4345

‎libbeat/autodiscover/providers/docker/docker.go‎

Lines changed: 80 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
package docker
1919

2020
import (
21+
"errors"
22+
2123
"github.com/gofrs/uuid"
2224

2325
"github.com/elastic/beats/libbeat/autodiscover"
@@ -119,41 +121,91 @@ func (d *Provider) Start() {
119121
}()
120122
}
121123

122-
func (d *Provider) emitContainer(event bus.Event, flag string) {
124+
type dockerMetadata struct {
125+
// Old selectors [Deprecated]
126+
Docker common.MapStr
127+
128+
// New ECS-based selectors
129+
Container common.MapStr
130+
131+
// Metadata used to enrich events, like ECS-based selectors but can
132+
// have modifications like dedotting
133+
Metadata common.MapStr
134+
}
135+
136+
func (d *Provider) generateMetaDocker(event bus.Event) (*docker.Container, *dockerMetadata) {
123137
container, ok := event["container"].(*docker.Container)
124138
if !ok {
125-
logp.Err("Couldn't get a container from watcher event")
126-
return
139+
logp.Error(errors.New("Couldn't get a container from watcher event"))
140+
return nil, nil
127141
}
128142

129-
var host string
130-
if len(container.IPAddresses) > 0 {
131-
host = container.IPAddresses[0]
132-
}
143+
// Don't dedot selectors, dedot only metadata used for events enrichment
133144
labelMap := common.MapStr{}
145+
metaLabelMap := common.MapStr{}
134146
for k, v := range container.Labels {
135147
safemapstr.Put(labelMap, k, v)
148+
if d.config.Dedot {
149+
label := common.DeDot(k)
150+
metaLabelMap.Put(label, v)
151+
} else {
152+
safemapstr.Put(metaLabelMap, k, v)
153+
}
136154
}
137155

138-
meta := common.MapStr{
139-
"container": common.MapStr{
140-
"id": container.ID,
141-
"name": container.Name,
142-
"image": container.Image,
156+
meta := &dockerMetadata{
157+
Docker: common.MapStr{
158+
"container": common.MapStr{
159+
"id": container.ID,
160+
"name": container.Name,
161+
"image": container.Image,
162+
"labels": labelMap,
163+
},
164+
},
165+
Container: common.MapStr{
166+
"id": container.ID,
167+
"name": container.Name,
168+
"image": common.MapStr{
169+
"name": container.Image,
170+
},
143171
"labels": labelMap,
144172
},
173+
Metadata: common.MapStr{
174+
"container": common.MapStr{
175+
"id": container.ID,
176+
"name": container.Name,
177+
"image": common.MapStr{
178+
"name": container.Image,
179+
},
180+
},
181+
"docker": common.MapStr{
182+
"container": common.MapStr{
183+
"labels": metaLabelMap,
184+
},
185+
},
186+
},
187+
}
188+
189+
return container, meta
190+
}
191+
192+
func (d *Provider) emitContainer(event bus.Event, flag string) {
193+
container, meta := d.generateMetaDocker(event)
194+
var host string
195+
if len(container.IPAddresses) > 0 {
196+
host = container.IPAddresses[0]
145197
}
198+
146199
// Without this check there would be overlapping configurations with and without ports.
147200
if len(container.Ports) == 0 {
148201
event := bus.Event{
149-
"provider": d.uuid,
150-
"id": container.ID,
151-
flag: true,
152-
"host": host,
153-
"docker": meta,
154-
"meta": common.MapStr{
155-
"docker": meta,
156-
},
202+
"provider": d.uuid,
203+
"id": container.ID,
204+
flag: true,
205+
"host": host,
206+
"docker": meta.Docker,
207+
"container": meta.Container,
208+
"meta": meta.Metadata,
157209
}
158210

159211
d.publish(event)
@@ -162,15 +214,14 @@ func (d *Provider) emitContainer(event bus.Event, flag string) {
162214
// Emit container container and port information
163215
for _, port := range container.Ports {
164216
event := bus.Event{
165-
"provider": d.uuid,
166-
"id": container.ID,
167-
flag: true,
168-
"host": host,
169-
"port": port.PrivatePort,
170-
"docker": meta,
171-
"meta": common.MapStr{
172-
"docker": meta,
173-
},
217+
"provider": d.uuid,
218+
"id": container.ID,
219+
flag: true,
220+
"host": host,
221+
"port": port.PrivatePort,
222+
"docker": meta.Docker,
223+
"container": meta.Container,
224+
"meta": meta.Metadata,
174225
}
175226

176227
d.publish(event)

‎libbeat/autodiscover/providers/docker/docker_integration_test.go‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,17 +92,23 @@ func checkEvent(t *testing.T, listener bus.Listener, start bool) {
9292
assert.Equal(t, getValue(e, "stop"), true)
9393
assert.Nil(t, getValue(e, "start"))
9494
}
95-
assert.Equal(t, getValue(e, "docker.container.image"), "busybox")
96-
assert.Equal(t, getValue(e, "docker.container.labels"), common.MapStr{
97-
"label": common.MapStr{
98-
"value": "foo",
99-
"child": "bar",
95+
assert.Equal(t, getValue(e, "container.image.name"), "busybox")
96+
// labels.dedot=true by default
97+
assert.Equal(t,
98+
common.MapStr{
99+
"label": common.MapStr{
100+
"value": "foo",
101+
"child": "bar",
102+
},
100103
},
101-
})
102-
assert.NotNil(t, getValue(e, "docker.container.id"))
103-
assert.NotNil(t, getValue(e, "docker.container.name"))
104+
getValue(e, "container.labels"),
105+
)
106+
assert.NotNil(t, getValue(e, "container.id"))
107+
assert.NotNil(t, getValue(e, "container.name"))
104108
assert.NotNil(t, getValue(e, "host"))
105-
assert.Equal(t, getValue(e, "docker"), getValue(e, "meta.docker"))
109+
assert.Equal(t, getValue(e, "docker.container.id"), getValue(e, "meta.container.id"))
110+
assert.Equal(t, getValue(e, "docker.container.name"), getValue(e, "meta.container.name"))
111+
assert.Equal(t, getValue(e, "docker.container.image"), getValue(e, "meta.container.image.name"))
106112
return
107113

108114
case <-time.After(10 * time.Second):

‎libbeat/autodiscover/providers/docker/docker_test.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424

2525
"github.com/elastic/beats/libbeat/common"
2626
"github.com/elastic/beats/libbeat/common/bus"
27+
"github.com/elastic/beats/libbeat/common/docker"
2728
)
2829

2930
func TestGenerateHints(t *testing.T) {
@@ -105,3 +106,131 @@ func getNestedAnnotations(in common.MapStr) common.MapStr {
105106
}
106107
return out
107108
}
109+
110+
func TestGenerateMetaDockerNoDedot(t *testing.T) {
111+
event := bus.Event{
112+
"container": &docker.Container{
113+
ID: "abc",
114+
Name: "foobar",
115+
Labels: map[string]string{
116+
"do.not.include": "true",
117+
"co.elastic.logs/disable": "true",
118+
},
119+
},
120+
}
121+
122+
cfg := defaultConfig()
123+
cfg.Dedot = false
124+
p := Provider{
125+
config: cfg,
126+
}
127+
_, meta := p.generateMetaDocker(event)
128+
expectedMeta := &dockerMetadata{
129+
Docker: common.MapStr{
130+
"container": common.MapStr{
131+
"id": "abc",
132+
"name": "foobar",
133+
"image": "",
134+
"labels": common.MapStr{
135+
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
136+
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
137+
},
138+
},
139+
},
140+
Container: common.MapStr{
141+
"id": "abc",
142+
"name": "foobar",
143+
"image": common.MapStr{
144+
"name": "",
145+
},
146+
"labels": common.MapStr{
147+
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
148+
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
149+
},
150+
},
151+
Metadata: common.MapStr{
152+
"container": common.MapStr{
153+
"id": "abc",
154+
"name": "foobar",
155+
"image": common.MapStr{
156+
"name": "",
157+
},
158+
},
159+
"docker": common.MapStr{
160+
"container": common.MapStr{
161+
"labels": common.MapStr{
162+
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
163+
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
164+
},
165+
},
166+
},
167+
},
168+
}
169+
assert.Equal(t, expectedMeta.Docker, meta.Docker)
170+
assert.Equal(t, expectedMeta.Container, meta.Container)
171+
assert.Equal(t, expectedMeta.Metadata, meta.Metadata)
172+
}
173+
174+
func TestGenerateMetaDockerWithDedot(t *testing.T) {
175+
event := bus.Event{
176+
"container": &docker.Container{
177+
ID: "abc",
178+
Name: "foobar",
179+
Labels: map[string]string{
180+
"do.not.include": "true",
181+
"co.elastic.logs/disable": "true",
182+
},
183+
},
184+
}
185+
186+
cfg := defaultConfig()
187+
cfg.Dedot = true
188+
p := Provider{
189+
config: cfg,
190+
}
191+
_, meta := p.generateMetaDocker(event)
192+
expectedMeta := &dockerMetadata{
193+
Docker: common.MapStr{
194+
"container": common.MapStr{
195+
"id": "abc",
196+
"name": "foobar",
197+
"image": "",
198+
"labels": common.MapStr{
199+
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
200+
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
201+
},
202+
},
203+
},
204+
Container: common.MapStr{
205+
"id": "abc",
206+
"name": "foobar",
207+
"image": common.MapStr{
208+
"name": "",
209+
},
210+
"labels": common.MapStr{
211+
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
212+
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
213+
},
214+
},
215+
Metadata: common.MapStr{
216+
"container": common.MapStr{
217+
"id": "abc",
218+
"name": "foobar",
219+
"image": common.MapStr{
220+
"name": "",
221+
},
222+
},
223+
"docker": common.MapStr{
224+
"container": common.MapStr{
225+
"labels": common.MapStr{
226+
"do_not_include": "true",
227+
"co_elastic_logs/disable": "true",
228+
},
229+
},
230+
},
231+
},
232+
}
233+
assert.Equal(t, expectedMeta.Docker, meta.Docker)
234+
assert.Equal(t, expectedMeta.Container, meta.Container)
235+
assert.Equal(t, expectedMeta.Metadata, meta.Metadata)
236+
}

‎metricbeat/tests/system/test_autodiscover.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,10 @@ def test_docker(self):
5151
proc.check_kill_and_wait()
5252

5353
# Check metadata is added
54-
assert output[0]['docker']['container']['image'] == 'memcached:latest'
54+
assert output[0]['container']['image']['name'] == 'memcached:latest'
5555
assert output[0]['docker']['container']['labels'] == {}
56-
assert 'name' in output[0]['docker']['container']
56+
assert 'name' in output[0]['container']
57+
self.assert_fields_are_documented(output[0])
5758

5859
@unittest.skipIf(not INTEGRATION_TESTS or
5960
os.getenv("TESTING_ENVIRONMENT") == "2x",
@@ -93,8 +94,9 @@ def test_docker_labels(self):
9394
proc.check_kill_and_wait()
9495

9596
# Check metadata is added
96-
assert output[0]['docker']['container']['image'] == 'memcached:latest'
97-
assert 'name' in output[0]['docker']['container']
97+
assert output[0]['container']['image']['name'] == 'memcached:latest'
98+
assert 'name' in output[0]['container']
99+
self.assert_fields_are_documented(output[0])
98100

99101
@unittest.skipIf(not INTEGRATION_TESTS or
100102
os.getenv("TESTING_ENVIRONMENT") == "2x",
@@ -143,3 +145,4 @@ def test_config_appender(self):
143145

144146
# Check field is added
145147
assert output[0]['fields']['foo'] == 'bar'
148+
self.assert_fields_are_documented(output[0])

0 commit comments

Comments
 (0)