Skip to content

Commit 75cb108

Browse files
ChrsMarkChrsMark
committed
Bring back add_resource_metadata section
Signed-off-by: chrismark <chrismarkou92@gmail.com>
1 parent ef89838 commit 75cb108

11 files changed

Lines changed: 85 additions & 16 deletions

File tree

‎CHANGELOG.next.asciidoc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
490490
- Add kubernetes.node.hostname metadata of Kubernetes node. {pull}22189[22189]
491491
- Include node metadata and namespace metadata in pod's metadata by default for autodiscovery. {pull}22189[22189]
492492
- Include node metadata and namespace metadata in pod's metadata by default for add_kuberentes_metadata. {pull}22189[22189]
493-
- Deprecate add_resource_metadata of kubernetes autodiscovery. {pull}22189[22189]
493+
- Enable always add_resource_metadata for Pods and Services of kubernetes autodiscovery. {pull}22189[22189]
494+
- Add add_resource_metadata option setting (always enabled) for add_kubernetes_metadata setting. {pull}22189[22189]
494495
- Added Kafka version 2.2 to the list of supported versions. {pull}22328[22328]
495496

496497
*Auditbeat*

‎filebeat/docs/autodiscover-hints.asciidoc‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,20 @@ tokenizer. The second input handles everything but debug logs.
175175
===== Namespace Defaults
176176

177177
Hints can be configured on the Namespace's annotations as defaults to use when Pod level annotations are missing.
178-
The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence.
178+
The resultant hints are a combination of Pod annotations and Namespace annotations with the Pod's taking precedence. To
179+
enable Namespace defaults configure the `add_resource_metadata` for Namespace objects as follows:
180+
181+
["source","yaml",subs="attributes"]
182+
-------------------------------------------------------------------------------------
183+
filebeat.autodiscover:
184+
providers:
185+
- type: kubernetes
186+
hints.enabled: true
187+
add_resource_metadata:
188+
namespace:
189+
include_annotations: ["nsannotation1"]
190+
-------------------------------------------------------------------------------------
191+
179192

180193

181194
[float]

‎libbeat/autodiscover/providers/kubernetes/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import (
2323
"fmt"
2424
"time"
2525

26+
"github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata"
27+
2628
"github.com/elastic/beats/v7/libbeat/autodiscover/template"
2729
"github.com/elastic/beats/v7/libbeat/common"
2830
"github.com/elastic/beats/v7/libbeat/common/cfgwarn"
@@ -51,6 +53,8 @@ type Config struct {
5153
Builders []*common.Config `config:"builders"`
5254
Appenders []*common.Config `config:"appenders"`
5355
Templates template.MapperSettings `config:"templates"`
56+
57+
AddResourceMetadata *metadata.AddResourceMetadataConfig `config:"add_resource_metadata"`
5458
}
5559

5660
func defaultConfig() *Config {

‎libbeat/autodiscover/providers/kubernetes/pod.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,10 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
8282
if config.Namespace != "" {
8383
options.Namespace = config.Namespace
8484
}
85-
85+
metaConf := config.AddResourceMetadata
86+
if metaConf == nil {
87+
metaConf = metadata.GetDefaultResourceMetadataConfig()
88+
}
8689
nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
8790
if err != nil {
8891
return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
@@ -93,7 +96,7 @@ func NewPodEventer(uuid uuid.UUID, cfg *common.Config, client k8s.Interface, pub
9396
if err != nil {
9497
return nil, fmt.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
9598
}
96-
metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher)
99+
metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, metaConf)
97100

98101
p := &pod{
99102
config: config,

‎libbeat/common/kubernetes/metadata/metadata.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,9 +57,9 @@ func GetPodMetaGen(
5757
cfg *common.Config,
5858
podWatcher kubernetes.Watcher,
5959
nodeWatcher kubernetes.Watcher,
60-
namespaceWatcher kubernetes.Watcher) MetaGen {
60+
namespaceWatcher kubernetes.Watcher,
61+
metaConf *AddResourceMetadataConfig) MetaGen {
6162

62-
metaConf := GetDefaultResourceMetadataConfig()
6363
nodeMetaGen := NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store())
6464
namespaceMetaGen := NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store())
6565
metaGen := NewPodMetadataGenerator(cfg, podWatcher.Store(), nodeMetaGen, namespaceMetaGen)

‎libbeat/common/kubernetes/metadata/resource.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,7 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie
6161
labelMap.Delete(label)
6262
}
6363

64-
annotationsMap := common.MapStr{}
65-
if len(r.config.IncludeAnnotations) == 0 {
66-
annotationsMap = generateMap(accessor.GetAnnotations(), r.config.LabelsDedot)
67-
} else {
68-
annotationsMap = generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.LabelsDedot)
69-
}
64+
annotationsMap := generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.LabelsDedot)
7065

7166
meta := common.MapStr{
7267
strings.ToLower(kind): common.MapStr{

‎libbeat/docs/shared-autodiscover.asciidoc‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,10 +229,26 @@ running configuration for a container, 60s by default.
229229
either take `node` or `cluster` as values. `node` scope allows discovery of resources in
230230
the specified node. `cluster` scope allows cluster wide discovery. Only `pod` and `node` resources
231231
can be discovered at node scope.
232+
`add_resource_metadata`:: (Optional) Specify labels and annotations filters for the extra metadata coming from Node and Namespace.
233+
`add_resource_metadata` can be done for `node` or `namespace`. By default all labels will be included
234+
while annotations are not added by default. This settings are useful when labels' and annotations'
235+
storing requires special handling to avoid overloading the storage output.
236+
Example:
237+
238+
["source","yaml",subs="attributes"]
239+
-------------------------------------------------------------------------------------
240+
add_resource_metadata:
241+
namespace:
242+
include_labels: ["namespacelabel1"]
243+
node:
244+
include_labels: ["nodelabel2"]
245+
include_annotations: ["nodeannotation1"]
246+
-------------------------------------------------------------------------------------
247+
232248
`unique`:: (Optional) Defaults to `false`. Marking an autodiscover provider as unique results into
233249
making the provider to enable the provided templates only when it will gain the leader lease.
234250
This setting can only be combined with `cluster` scope. When `unique` is enabled enabled, `resource`
235-
setting is not taken into account.
251+
and `add_resource_metadata` settings are not taken into account.
236252
`leader_lease`:: (Optional) Defaults to `{beatname_lc}-cluster-leader`. This will be name of the lock lease.
237253
One can monitor the status of the lease with `kubectl describe lease beats-cluster-leader`.
238254
Different Beats that refer to the same leader lease will be competetitors in holding the lease

‎libbeat/processors/add_kubernetes_metadata/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"time"
2222

2323
"github.com/elastic/beats/v7/libbeat/common"
24+
"github.com/elastic/beats/v7/libbeat/common/kubernetes/metadata"
2425
)
2526

2627
type kubeAnnotatorConfig struct {
@@ -35,6 +36,8 @@ type kubeAnnotatorConfig struct {
3536
Matchers PluginConfig `config:"matchers"`
3637
DefaultMatchers Enabled `config:"default_matchers"`
3738
DefaultIndexers Enabled `config:"default_indexers"`
39+
40+
AddResourceMetadata *metadata.AddResourceMetadataConfig `config:"add_resource_metadata"`
3841
}
3942

4043
type Enabled struct {

‎libbeat/processors/add_kubernetes_metadata/docs/add_kubernetes_metadata.asciidoc‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ Each event is annotated with:
1919
* Namespace
2020
* Labels
2121

22+
In addition Node's metadata and Namespace's metadata are being added in Pod's metadata.
23+
2224
The `add_kubernetes_metadata` processor has two basic building blocks which are:
2325

2426
* Indexers
@@ -116,6 +118,21 @@ mode.
116118
`namespace`:: (Optional) Select the namespace from which to collect the
117119
metadata. If it is not set, the processor collects metadata from all namespaces.
118120
It is unset by default.
121+
`add_resource_metadata`:: (Optional) Specify labels and annotations filters for the extra metadata coming from Node and Namespace.
122+
`add_resource_metadata` can be done for `node` or `namespace`. By default all labels will be included
123+
while annotations are not added by default. This settings are useful when labels' and annotations'
124+
storing requires special handling to avoid overloading the storage output.
125+
Example:
126+
127+
["source","yaml",subs="attributes"]
128+
-------------------------------------------------------------------------------------
129+
add_resource_metadata:
130+
namespace:
131+
include_labels: ["namespacelabel1"]
132+
node:
133+
include_labels: ["nodelabel2"]
134+
include_annotations: ["nodeannotation1"]
135+
-------------------------------------------------------------------------------------
119136
`kube_config`:: (Optional) Use given config file as configuration for Kubernetes
120137
client. It defaults to `KUBECONFIG` environment variable if present.
121138
`default_indexers.enabled`:: (Optional) Enable/Disable default pod indexers, in

‎libbeat/processors/add_kubernetes_metadata/kubernetes.go‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,14 +171,18 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Confi
171171
return
172172
}
173173

174+
metaConf := config.AddResourceMetadata
175+
if metaConf == nil {
176+
metaConf = metadata.GetDefaultResourceMetadataConfig()
177+
}
178+
174179
options := kubernetes.WatchOptions{
175180
SyncTimeout: config.SyncPeriod,
176181
Node: "",
177182
}
178183
if config.Namespace != "" {
179184
options.Namespace = config.Namespace
180185
}
181-
182186
nodeWatcher, err := kubernetes.NewWatcher(client, &kubernetes.Node{}, options, nil)
183187
if err != nil {
184188
k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Node{}, err)
@@ -190,7 +194,7 @@ func (k *kubernetesAnnotator) init(config kubeAnnotatorConfig, cfg *common.Confi
190194
k.log.Errorf("couldn't create watcher for %T due to error %+v", &kubernetes.Namespace{}, err)
191195
}
192196
// TODO: refactor the above section to a common function to be used by NeWPodEventer too
193-
metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher)
197+
metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, metaConf)
194198

195199
k.indexers = NewIndexers(config.Indexers, metaGen)
196200
k.watcher = watcher

0 commit comments

Comments
 (0)