Skip to content

Commit ff67957

Browse files
tetianakravchenkotetianakravchenko
authored
Align k8s metadata configurations in Kubernetes module: add addResourceMetadata config (elastic#29133)
* use addResourceMetadata config instead of hardcoded bool Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * revert config for local dev Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * update doc; use common function to create all watchers; add nodeWatcher and nsWatcher to enricher Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * revert ek_stack Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * add stop watchers; adjust test for dedoting labels; fix overriding config that leads to 'failed to parse field' error Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * adjust log messages; check that watcher is not nil Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * fix error message format; rename getPodMetadataWatchers -> getResourceMetadataWatchers; return resource watcher if nodeWatcher/nsWatcher failed to be created Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co> * add changelog Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>
1 parent c7e43b6 commit ff67957

19 files changed

Lines changed: 414 additions & 112 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
371371
- Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
372372
- Update reference to gosigar pacakge for filesystem windows fix. {pull}28909[28909]
373373
- Override `Host()` on statsd MetricSet {pull}29103[29103]
374+
- Add `add_resource_metadata` configuration to Kubernetes module. {pull}29133[29133]
374375

375376
*Packetbeat*
376377

‎libbeat/autodiscover/providers/kubernetes/pod.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,9 +100,6 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
100100
options.Namespace = config.Namespace
101101
}
102102
metaConf := config.AddResourceMetadata
103-
if metaConf == nil {
104-
metaConf = metadata.GetDefaultResourceMetadataConfig()
105-
}
106103
nodeWatcher, err := kubernetes.NewNamedWatcher("node", client, &kubernetes.Node{}, options, nil)
107104
if err != nil {
108105
logger.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)

‎libbeat/autodiscover/providers/kubernetes/pod_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1907,7 +1907,7 @@ func TestPod_EmitEvent(t *testing.T) {
19071907
t.Fatal(err)
19081908
}
19091909

1910-
metaGen := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, client, nil, nil, true)
1910+
metaGen := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, client, nil, nil, nil)
19111911
p := &Provider{
19121912
config: defaultConfig(),
19131913
bus: bus.New(logp.NewLogger("bus"), "test"),

‎libbeat/common/kubernetes/metadata/metadata.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,7 @@ func GetPodMetaGen(
9696
if namespaceWatcher != nil && metaConf.Namespace.Enabled() {
9797
namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client())
9898
}
99-
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, metaConf.Deployment)
100-
99+
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), podWatcher.Client(), nodeMetaGen, namespaceMetaGen, metaConf)
101100
return metaGen
102101
}
103102

‎libbeat/common/kubernetes/metadata/namespace_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,11 @@ func TestNamespace_Generate(t *testing.T) {
5151
UID: types.UID(uid),
5252
Labels: map[string]string{
5353
"foo": "bar",
54+
"key": "value",
5455
},
5556
Annotations: map[string]string{
5657
"spam": "baz",
58+
"key": "value",
5759
},
5860
},
5961
TypeMeta: metav1.TypeMeta{
@@ -75,6 +77,7 @@ func TestNamespace_Generate(t *testing.T) {
7577
}
7678

7779
cfg, err := common.NewConfigFrom(Config{
80+
IncludeLabels: []string{"foo"},
7881
IncludeAnnotations: []string{"spam"},
7982
})
8083
if err != nil {

‎libbeat/common/kubernetes/metadata/pod.go‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,12 @@ import (
2929
)
3030

3131
type pod struct {
32-
store cache.Store
33-
client k8s.Interface
34-
node MetaGen
35-
namespace MetaGen
36-
resource *Resource
37-
addDeployment bool
32+
store cache.Store
33+
client k8s.Interface
34+
node MetaGen
35+
namespace MetaGen
36+
resource *Resource
37+
addResourceMetadata *AddResourceMetadataConfig
3838
}
3939

4040
// NewPodMetadataGenerator creates a metagen for pod resources
@@ -44,14 +44,19 @@ func NewPodMetadataGenerator(
4444
client k8s.Interface,
4545
node MetaGen,
4646
namespace MetaGen,
47-
addDeploymentMeta bool) MetaGen {
47+
addResourceMetadata *AddResourceMetadataConfig) MetaGen {
48+
49+
if addResourceMetadata == nil {
50+
addResourceMetadata = GetDefaultResourceMetadataConfig()
51+
}
52+
4853
return &pod{
49-
resource: NewResourceMetadataGenerator(cfg, client),
50-
store: pods,
51-
node: node,
52-
namespace: namespace,
53-
client: client,
54-
addDeployment: addDeploymentMeta,
54+
resource: NewResourceMetadataGenerator(cfg, client),
55+
store: pods,
56+
node: node,
57+
namespace: namespace,
58+
client: client,
59+
addResourceMetadata: addResourceMetadata,
5560
}
5661
}
5762

@@ -87,7 +92,7 @@ func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) common.
8792
out := p.resource.GenerateK8s("pod", obj, opts...)
8893

8994
// check if Pod is handled by a ReplicaSet which is controlled by a Deployment
90-
if p.addDeployment {
95+
if p.addResourceMetadata.Deployment {
9196
rsName, _ := out.GetValue("replicaset.name")
9297
if rsName, ok := rsName.(string); ok {
9398
dep := p.getRSDeployment(rsName, po.GetNamespace())

‎libbeat/common/kubernetes/metadata/pod_test.go‎

Lines changed: 152 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ func TestPod_Generate(t *testing.T) {
374374
})
375375
assert.NoError(t, err)
376376

377-
metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, true)
377+
metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, nil)
378378
for _, test := range tests {
379379
t.Run(test.name, func(t *testing.T) {
380380
assert.Equal(t, test.output, metagen.Generate(test.input))
@@ -496,7 +496,7 @@ func TestPod_GenerateFromName(t *testing.T) {
496496
assert.NoError(t, err)
497497
pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
498498
pods.Add(test.input)
499-
metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, true)
499+
metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, nil)
500500

501501
accessor, err := meta.Accessor(test.input)
502502
require.NoError(t, err)
@@ -618,7 +618,156 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) {
618618
namespaces.Add(test.namespace)
619619
nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client)
620620

621-
metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, true)
621+
metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, nil)
622+
t.Run(test.name, func(t *testing.T) {
623+
assert.Equal(t, test.output, metagen.Generate(test.input))
624+
})
625+
}
626+
}
627+
628+
func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) {
629+
client := k8sfake.NewSimpleClientset()
630+
uid := "005f3b90-4b9d-12f8-acf0-31020a840133"
631+
namespace := "default"
632+
name := "obj"
633+
boolean := true
634+
635+
tests := []struct {
636+
input kubernetes.Resource
637+
node kubernetes.Resource
638+
namespace kubernetes.Resource
639+
output common.MapStr
640+
name string
641+
}{
642+
{
643+
name: "test simple object",
644+
input: &v1.Pod{
645+
ObjectMeta: metav1.ObjectMeta{
646+
Name: name,
647+
UID: types.UID(uid),
648+
Namespace: namespace,
649+
Labels: map[string]string{
650+
"app.kubernetes.io/component": "exporter",
651+
},
652+
Annotations: map[string]string{
653+
"app": "production",
654+
},
655+
OwnerReferences: []metav1.OwnerReference{
656+
{
657+
APIVersion: "apps",
658+
Kind: "ReplicaSet",
659+
Name: "nginx-rs",
660+
UID: "005f3b90-4b9d-12f8-acf0-31020a8409087",
661+
Controller: &boolean,
662+
},
663+
},
664+
},
665+
TypeMeta: metav1.TypeMeta{
666+
Kind: "Pod",
667+
APIVersion: "v1",
668+
},
669+
670+
Spec: v1.PodSpec{
671+
NodeName: "testnode",
672+
},
673+
Status: v1.PodStatus{PodIP: "127.0.0.5"},
674+
},
675+
node: &v1.Node{
676+
ObjectMeta: metav1.ObjectMeta{
677+
Name: "testnode",
678+
UID: types.UID(uid),
679+
Labels: map[string]string{
680+
"nodekey": "nodevalue",
681+
"nodekey2": "nodevalue2",
682+
},
683+
Annotations: map[string]string{},
684+
},
685+
TypeMeta: metav1.TypeMeta{
686+
Kind: "Node",
687+
APIVersion: "v1",
688+
},
689+
Status: v1.NodeStatus{
690+
Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}},
691+
},
692+
},
693+
namespace: &v1.Namespace{
694+
ObjectMeta: metav1.ObjectMeta{
695+
Name: namespace,
696+
UID: types.UID(uid),
697+
Labels: map[string]string{
698+
"app.kubernetes.io/name": "kube-state-metrics",
699+
"nskey2": "nsvalue2",
700+
},
701+
Annotations: map[string]string{},
702+
},
703+
TypeMeta: metav1.TypeMeta{
704+
Kind: "Namespace",
705+
APIVersion: "v1",
706+
},
707+
},
708+
output: common.MapStr{"kubernetes": common.MapStr{
709+
"pod": common.MapStr{
710+
"name": "obj",
711+
"uid": uid,
712+
"ip": "127.0.0.5",
713+
},
714+
"namespace": "default",
715+
"namespace_uid": uid,
716+
"namespace_labels": common.MapStr{
717+
"app_kubernetes_io/name": "kube-state-metrics",
718+
},
719+
"node": common.MapStr{
720+
"name": "testnode",
721+
"uid": uid,
722+
"labels": common.MapStr{
723+
"nodekey2": "nodevalue2",
724+
},
725+
"hostname": "node1",
726+
},
727+
"labels": common.MapStr{
728+
"app_kubernetes_io/component": "exporter",
729+
},
730+
"annotations": common.MapStr{
731+
"app": "production",
732+
},
733+
"replicaset": common.MapStr{
734+
"name": "nginx-rs",
735+
},
736+
}},
737+
},
738+
}
739+
740+
for _, test := range tests {
741+
config, err := common.NewConfigFrom(map[string]interface{}{
742+
"include_annotations": []string{"app"},
743+
})
744+
745+
assert.NoError(t, err)
746+
747+
namespaceConfig, _ := common.NewConfigFrom(map[string]interface{}{
748+
"include_labels": []string{"app.kubernetes.io/name"},
749+
})
750+
nodeConfig, _ := common.NewConfigFrom(map[string]interface{}{
751+
"include_labels": []string{"nodekey2"},
752+
})
753+
metaConfig := AddResourceMetadataConfig{
754+
Namespace: namespaceConfig,
755+
Node: nodeConfig,
756+
Deployment: false,
757+
}
758+
759+
pods := cache.NewStore(cache.MetaNamespaceKeyFunc)
760+
pods.Add(test.input)
761+
762+
nodes := cache.NewStore(cache.MetaNamespaceKeyFunc)
763+
nodes.Add(test.node)
764+
nodeMeta := NewNodeMetadataGenerator(nodeConfig, nodes, client)
765+
766+
namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc)
767+
namespaces.Add(test.namespace)
768+
nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client)
769+
770+
metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, &metaConfig)
622771
t.Run(test.name, func(t *testing.T) {
623772
assert.Equal(t, test.output, metagen.Generate(test.input))
624773
})

‎libbeat/common/kubernetes/metadata/resource.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ...
8585
return nil
8686
}
8787

88-
labelMap := common.MapStr{}
88+
var labelMap common.MapStr
8989
if len(r.config.IncludeLabels) == 0 {
9090
labelMap = GenerateMap(accessor.GetLabels(), r.config.LabelsDedot)
9191
} else {

‎libbeat/processors/add_kubernetes_metadata/indexers_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import (
3232
"github.com/elastic/beats/v7/libbeat/common/kubernetes"
3333
)
3434

35-
var metagen = metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, true)
35+
var metagen = metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, nil)
3636

3737
func TestPodIndexer(t *testing.T) {
3838
var testConfig = common.NewConfig()
@@ -90,7 +90,7 @@ func TestPodIndexer(t *testing.T) {
9090
func TestPodUIDIndexer(t *testing.T) {
9191
var testConfig = common.NewConfig()
9292

93-
metaGenWithPodUID := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, true)
93+
metaGenWithPodUID := metadata.NewPodMetadataGenerator(common.NewConfig(), nil, nil, nil, nil, nil)
9494

9595
podUIDIndexer, err := NewPodUIDIndexer(*testConfig, metaGenWithPodUID)
9696
assert.NoError(t, err)
@@ -301,7 +301,7 @@ func TestFilteredGenMeta(t *testing.T) {
301301
})
302302
assert.NoError(t, err)
303303

304-
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, true)
304+
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, nil)
305305

306306
podIndexer, err = NewPodNameIndexer(*testConfig, filteredGen)
307307
assert.NoError(t, err)
@@ -338,7 +338,7 @@ func TestFilteredGenMetaExclusion(t *testing.T) {
338338
})
339339
assert.NoError(t, err)
340340

341-
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, true)
341+
filteredGen := metadata.NewPodMetadataGenerator(config, nil, nil, nil, nil, nil)
342342

343343
podIndexer, err := NewPodNameIndexer(*testConfig, filteredGen)
344344
assert.NoError(t, err)

‎libbeat/processors/add_kubernetes_metadata/kubernetes.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,9 +191,6 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Confi
191191
}
192192

193193
metaConf := config.AddResourceMetadata
194-
if metaConf == nil {
195-
metaConf = metadata.GetDefaultResourceMetadataConfig()
196-
}
197194

198195
options := kubernetes.WatchOptions{
199196
SyncTimeout: config.SyncPeriod,

0 commit comments

Comments
 (0)