Skip to content

Commit 36cee51

Browse files
ruflinruflin
authored
Fix issue with ILM that write index was created before template was loaded (#9617)
The write index for ILM was created before the template was loaded. This meant the first template did not contain all the mappings and policy for ILM so was never rolled over. This change makes sure the template is loaded for ILM is setup up and adds a test to confirm it. Further ILM was not respected when running `metricbeat setup --template`. This is now changed too.
1 parent 8eff2a8 commit 36cee51

3 files changed

Lines changed: 82 additions & 45 deletions

File tree

‎libbeat/cmd/instance/beat.go‎

Lines changed: 62 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,20 @@ func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning
456456
return fmt.Errorf("Template loading requested but the Elasticsearch output is not configured/enabled")
457457
}
458458

459+
if b.Config.ILM.Enabled() {
460+
cfgwarn.Beta("Index lifecycle management is enabled which is in beta.")
461+
462+
ilmCfg, err := getILMConfig(b)
463+
if err != nil {
464+
return err
465+
}
466+
467+
err = b.prepareILMTemplate(ilmCfg)
468+
if err != nil {
469+
return err
470+
}
471+
}
472+
459473
esConfig := outCfg.Config()
460474
if tmplCfg := b.Config.Template; tmplCfg == nil || tmplCfg.Enabled() {
461475
loadCallback, err := b.templateLoadingCallback()
@@ -757,41 +771,30 @@ func (b *Beat) registerTemplateLoading() error {
757771
return errors.New("setup.template.name and setup.template.pattern have to be set if index name is modified")
758772
}
759773

760-
if b.Config.ILM.Enabled() {
761-
cfgwarn.Beta("Index lifecycle management is enabled which is in beta.")
774+
if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) {
762775

763-
ilmCfg, err := getILMConfig(b)
776+
// load template through callback to make sure it is also loaded
777+
// on reconnecting
778+
callback, err := b.templateLoadingCallback()
764779
if err != nil {
765780
return err
766781
}
782+
elasticsearch.RegisterConnectCallback(callback)
783+
} else if b.Config.ILM.Enabled() {
784+
return errors.New("templates cannot be disable when using ILM")
785+
}
767786

768-
// In case no template settings are set, config must be created
769-
if b.Config.Template == nil {
770-
b.Config.Template = common.NewConfig()
771-
}
772-
// Template name and pattern can't be configure when using ILM
773-
logp.Info("Set setup.template.name to '%s' as ILM is enabled.", ilmCfg.RolloverAlias)
774-
err = b.Config.Template.SetString("name", -1, ilmCfg.RolloverAlias)
775-
if err != nil {
776-
return errw.Wrap(err, "error setting setup.template.name")
777-
}
778-
pattern := fmt.Sprintf("%s-*", ilmCfg.RolloverAlias)
779-
logp.Info("Set setup.template.pattern to '%s' as ILM is enabled.", pattern)
780-
err = b.Config.Template.SetString("pattern", -1, pattern)
781-
if err != nil {
782-
return errw.Wrap(err, "error setting setup.template.pattern")
783-
}
787+
if b.Config.ILM.Enabled() {
788+
cfgwarn.Beta("Index lifecycle management is enabled which is in beta.")
784789

785-
// rollover_alias and lifecycle.name can't be configured and will be overwritten
786-
logp.Info("Set settings.index.lifecycle.rollover_alias in template to %s as ILM is enabled.", ilmCfg.RolloverAlias)
787-
err = b.Config.Template.SetString("settings.index.lifecycle.rollover_alias", -1, ilmCfg.RolloverAlias)
790+
ilmCfg, err := getILMConfig(b)
788791
if err != nil {
789-
return errw.Wrap(err, "error setting settings.index.lifecycle.rollover_alias")
792+
return err
790793
}
791-
logp.Info("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", ILMPolicyName)
792-
err = b.Config.Template.SetString("settings.index.lifecycle.name", -1, ILMPolicyName)
794+
795+
err = b.prepareILMTemplate(ilmCfg)
793796
if err != nil {
794-
return errw.Wrap(err, "error setting settings.index.lifecycle.name")
797+
return err
795798
}
796799

797800
// Set the ingestion index to the rollover alias
@@ -828,19 +831,39 @@ func (b *Beat) registerTemplateLoading() error {
828831

829832
elasticsearch.RegisterConnectCallback(writeAliasCallback)
830833
}
834+
}
831835

832-
if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) {
836+
return nil
837+
}
833838

834-
// load template through callback to make sure it is also loaded
835-
// on reconnecting
836-
callback, err := b.templateLoadingCallback()
837-
if err != nil {
838-
return err
839-
}
840-
elasticsearch.RegisterConnectCallback(callback)
841-
} else if b.Config.ILM.Enabled() {
842-
return errors.New("templates cannot be disable when using ILM")
843-
}
839+
func (b *Beat) prepareILMTemplate(ilmCfg *ilmConfig) error {
840+
// In case no template settings are set, config must be created
841+
if b.Config.Template == nil {
842+
b.Config.Template = common.NewConfig()
843+
}
844+
// Template name and pattern can't be configure when using ILM
845+
logp.Info("Set setup.template.name to '%s' as ILM is enabled.", ilmCfg.RolloverAlias)
846+
err := b.Config.Template.SetString("name", -1, ilmCfg.RolloverAlias)
847+
if err != nil {
848+
return errw.Wrap(err, "error setting setup.template.name")
849+
}
850+
pattern := fmt.Sprintf("%s-*", ilmCfg.RolloverAlias)
851+
logp.Info("Set setup.template.pattern to '%s' as ILM is enabled.", pattern)
852+
err = b.Config.Template.SetString("pattern", -1, pattern)
853+
if err != nil {
854+
return errw.Wrap(err, "error setting setup.template.pattern")
855+
}
856+
857+
// rollover_alias and lifecycle.name can't be configured and will be overwritten
858+
logp.Info("Set settings.index.lifecycle.rollover_alias in template to %s as ILM is enabled.", ilmCfg.RolloverAlias)
859+
err = b.Config.Template.SetString("settings.index.lifecycle.rollover_alias", -1, ilmCfg.RolloverAlias)
860+
if err != nil {
861+
return errw.Wrap(err, "error setting settings.index.lifecycle.rollover_alias")
862+
}
863+
logp.Info("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", ILMPolicyName)
864+
err = b.Config.Template.SetString("settings.index.lifecycle.name", -1, ILMPolicyName)
865+
if err != nil {
866+
return errw.Wrap(err, "error setting settings.index.lifecycle.name")
844867
}
845868

846869
return nil
@@ -863,6 +886,8 @@ func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) e
863886
return fmt.Errorf("Error loading Elasticsearch template: %v", err)
864887
}
865888

889+
logp.Info("Template successfully loaded.")
890+
866891
return nil
867892
}
868893

‎libbeat/mock/mockbeat.go‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,25 @@ func (mb *Mockbeat) Run(b *beat.Beat) error {
4949
return err
5050
}
5151

52-
// Wait until mockbeat is done
53-
go client.Publish(beat.Event{
54-
Timestamp: time.Now(),
55-
Fields: common.MapStr{
56-
"type": "mock",
57-
"message": "Mockbeat is alive!",
58-
},
59-
})
52+
ticker := time.NewTicker(1 * time.Second)
53+
go func() {
54+
for {
55+
select {
56+
case <-ticker.C:
57+
client.Publish(beat.Event{
58+
Timestamp: time.Now(),
59+
Fields: common.MapStr{
60+
"type": "mock",
61+
"message": "Mockbeat is alive!",
62+
},
63+
})
64+
case <-mb.done:
65+
ticker.Stop()
66+
return
67+
}
68+
}
69+
}()
70+
6071
<-mb.done
6172
return nil
6273
}

‎libbeat/tests/system/test_ilm.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ def test_setup_ilm_policy(self):
198198
policy = self.es.transport.perform_request('GET', "/_ilm/policy/" + self.policy_name)
199199
assert self.policy_name in policy
200200

201+
@unittest.skipUnless(INTEGRATION_TESTS, "integration test")
201202
@attr('integration')
202203
def test_export_ilm_policy(self):
203204
"""

0 commit comments

Comments
 (0)