Skip to content

Commit f72a6c0

Browse files
kvchmergify-bot
kvch
authored and
mergify-bot
committed
Import the references of dashboard assets using the Saved Objects API (#27647)
## What does this PR do? This PR changes the dashboard loading by first loading its references and then loading the dashboards. ## Why is it important? If the assets are not loaded in the proper order, the import can fail with unknown references. (cherry picked from commit b90370d)
1 parent 4bd11aa commit f72a6c0

5 files changed

Lines changed: 83 additions & 2 deletions

File tree

‎dev-tools/mage/kibana.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ func KibanaDashboards(moduleDirs ...string) error {
6969
// Convert 7.x dashboards to strings.
7070
err = sh.Run(pythonExe,
7171
filepath.Join(esBeatsDir, "libbeat/scripts/unpack_dashboards.py"),
72-
"--glob="+filepath.Join(kibanaBuildDir, "7/*/*.json"))
72+
"--glob="+filepath.Join(kibanaBuildDir, "7/dashboards/*.json"))
7373
if err != nil {
7474
return err
7575
}

‎libbeat/dashboards/importer.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ func (imp Importer) ImportDir(dirType string, dir string) error {
112112

113113
var errors []string
114114

115-
files, err := filepath.Glob(path.Join(dir, "*", "*.json"))
115+
files, err := filepath.Glob(path.Join(dir, dirType, "*.json"))
116116
if err != nil {
117117
return fmt.Errorf("Failed to read directory %s. Error: %s", dir, err)
118118
}

‎libbeat/dashboards/kibana_loader.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,11 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
151151

152152
content = ReplaceStringInDashboard("CHANGEME_HOSTNAME", loader.hostname, content)
153153

154+
err = loader.importReferences(file, content)
155+
if err != nil {
156+
return fmt.Errorf("error loading references of dashboard: %+v", err)
157+
}
158+
154159
var obj common.MapStr
155160
err = json.Unmarshal(content, &obj)
156161
if err != nil {
@@ -163,6 +168,35 @@ func (loader KibanaLoader) ImportDashboard(file string) error {
163168
return nil
164169
}
165170

171+
type dashboardObj struct {
172+
References []dashboardReference `json:"references"`
173+
}
174+
type dashboardReference struct {
175+
ID string `json:"id"`
176+
Type string `json:"type"`
177+
}
178+
179+
func (loader KibanaLoader) importReferences(path string, dashboard []byte) error {
180+
var d dashboardObj
181+
err := json.Unmarshal(dashboard, &d)
182+
if err != nil {
183+
return fmt.Errorf("failed to parse dashboard references: %+v", err)
184+
}
185+
186+
base := filepath.Dir(path)
187+
for _, ref := range d.References {
188+
if ref.Type == "index-pattern" {
189+
continue
190+
}
191+
referencePath := filepath.Join(base, "..", ref.Type, ref.ID+".json")
192+
err := loader.ImportDashboard(referencePath)
193+
if err != nil {
194+
return fmt.Errorf("error loading reference of %s: %s %s: %+v", path, ref.Type, ref.ID, err)
195+
}
196+
}
197+
return nil
198+
}
199+
166200
func correctExtension(file string) string {
167201
return filepath.Base(file[:len(file)-len("json")]) + "ndjson"
168202
}

‎libbeat/kibana/client.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,36 @@ func extractError(result []byte) error {
9191
return nil
9292
}
9393

94+
func extractMessage(result []byte) error {
95+
var kibanaResult struct {
96+
Success bool
97+
Errors []struct {
98+
Id string
99+
Type string
100+
Error struct {
101+
Type string
102+
References []struct {
103+
Type string
104+
Id string
105+
}
106+
}
107+
}
108+
}
109+
if err := json.Unmarshal(result, &kibanaResult); err != nil {
110+
return nil
111+
}
112+
113+
if !kibanaResult.Success {
114+
var errs multierror.Errors
115+
for _, err := range kibanaResult.Errors {
116+
errs = append(errs, fmt.Errorf("error: %s, asset ID=%s; asset type=%s; references=%+v", err.Error.Type, err.Id, err.Type, err.Error.References))
117+
}
118+
return errs.Err()
119+
}
120+
121+
return nil
122+
}
123+
94124
// NewKibanaClient builds and returns a new Kibana client
95125
func NewKibanaClient(cfg *common.Config) (*Client, error) {
96126
config := DefaultClientConfig()
@@ -186,6 +216,8 @@ func (conn *Connection) Request(method, extraPath string,
186216

187217
if resp.StatusCode >= 300 {
188218
retError = extractError(result)
219+
} else {
220+
retError = extractMessage(result)
189221
}
190222
return resp.StatusCode, result, retError
191223
}

‎libbeat/kibana/client_test.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,21 @@ func TestErrorBadJson(t *testing.T) {
6363
assert.Error(t, err)
6464
}
6565

66+
func TestErrorJsonWithHTTPOK(t *testing.T) {
67+
kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
68+
w.Write([]byte(`{"successCount":0,"success":false,"warnings":[],"errors":[{"id":"abcf35b0-0a82-11e8-bffe-ff7d4f68cf94-ecs","type":"dashboard","title":"[Filebeat MongoDB] Overview ECS","meta":{"title":"[Filebeat MongoDB] Overview ECS","icon":"dashboardApp"},"error":{"type":"missing_references","references":[{"type":"search","id":"e49fe000-0a7e-11e8-bffe-ff7d4f68cf94-ecs"},{"type":"search","id":"bfc96a60-0a80-11e8-bffe-ff7d4f68cf94-ecs"}]}}]}`))
69+
}))
70+
defer kibanaTs.Close()
71+
72+
conn := Connection{
73+
URL: kibanaTs.URL,
74+
HTTP: http.DefaultClient,
75+
}
76+
code, _, err := conn.Request(http.MethodPost, "", url.Values{}, nil, nil)
77+
assert.Equal(t, http.StatusOK, code)
78+
assert.Error(t, err)
79+
}
80+
6681
func TestSuccess(t *testing.T) {
6782
kibanaTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
6883
w.Write([]byte(`{"objects":[{"id":"test-*","type":"index-pattern","updated_at":"2018-01-24T19:04:13.371Z","version":1}]}`))

0 commit comments

Comments
 (0)