Skip to content
This repository was archived by the owner on Sep 21, 2023. It is now read-only.

Commit 4d8899c

Browse files
fearful-symmetryfearful-symmetry
authored
Update shipper to use fastjson, new shipper client version (#249)
* update shipper to use fastjson * linter * fix notice
1 parent 58a1029 commit 4d8899c

5 files changed

Lines changed: 100 additions & 50 deletions

File tree

‎NOTICE.txt‎

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -329,11 +329,11 @@ SOFTWARE
329329

330330
--------------------------------------------------------------------------------
331331
Dependency : github.com/elastic/elastic-agent-libs
332-
Version: v0.2.16
332+
Version: v0.3.0
333333
Licence type (autodetected): Apache-2.0
334334
--------------------------------------------------------------------------------
335335

336-
Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.2.16/LICENSE:
336+
Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.3.0/LICENSE:
337337

338338
Apache License
339339
Version 2.0, January 2004
@@ -540,11 +540,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-l
540540

541541
--------------------------------------------------------------------------------
542542
Dependency : github.com/elastic/elastic-agent-shipper-client
543-
Version: v0.4.1-0.20221028153110-a7eedbe6bd6c
543+
Version: v0.5.0
544544
Licence type (autodetected): Elastic
545545
--------------------------------------------------------------------------------
546546

547-
Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-shipper-client@v0.4.1-0.20221028153110-a7eedbe6bd6c/LICENSE.txt:
547+
Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-shipper-client@v0.5.0/LICENSE.txt:
548548

549549
Elastic License 2.0
550550

@@ -1331,6 +1331,39 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
13311331
SOFTWARE.
13321332

13331333

1334+
--------------------------------------------------------------------------------
1335+
Dependency : go.elastic.co/fastjson
1336+
Version: v1.1.0
1337+
Licence type (autodetected): MIT
1338+
--------------------------------------------------------------------------------
1339+
1340+
Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE:
1341+
1342+
Copyright 2018 Elasticsearch BV
1343+
1344+
Licensed under the Apache License, Version 2.0 (the "License");
1345+
you may not use this file except in compliance with the License.
1346+
You may obtain a copy of the License at
1347+
1348+
http://www.apache.org/licenses/LICENSE-2.0
1349+
1350+
Unless required by applicable law or agreed to in writing, software
1351+
distributed under the License is distributed on an "AS IS" BASIS,
1352+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1353+
See the License for the specific language governing permissions and
1354+
limitations under the License.
1355+
1356+
---
1357+
1358+
Copyright (c) 2016 Mail.Ru Group
1359+
1360+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
1361+
1362+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
1363+
1364+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
1365+
1366+
13341367
--------------------------------------------------------------------------------
13351368
Dependency : go.elastic.co/go-licence-detector
13361369
Version: v0.5.0
@@ -40116,11 +40149,11 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
4011640149

4011740150
--------------------------------------------------------------------------------
4011840151
Dependency : github.com/mitchellh/hashstructure
40119-
Version: v0.0.0-20170116052023-ab25296c0f51
40152+
Version: v1.1.0
4012040153
Licence type (autodetected): MIT
4012140154
--------------------------------------------------------------------------------
4012240155

40123-
Contents of probable licence file $GOMODCACHE/github.com/mitchellh/hashstructure@v0.0.0-20170116052023-ab25296c0f51/LICENSE:
40156+
Contents of probable licence file $GOMODCACHE/github.com/mitchellh/hashstructure@v1.1.0/LICENSE:
4012440157

4012540158
The MIT License (MIT)
4012640159

@@ -47448,39 +47481,6 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v1.0.1/LICENS
4744847481
See the License for the specific language governing permissions and
4744947482
limitations under the License.
4745047483

47451-
--------------------------------------------------------------------------------
47452-
Dependency : go.elastic.co/fastjson
47453-
Version: v1.1.0
47454-
Licence type (autodetected): MIT
47455-
--------------------------------------------------------------------------------
47456-
47457-
Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE:
47458-
47459-
Copyright 2018 Elasticsearch BV
47460-
47461-
Licensed under the Apache License, Version 2.0 (the "License");
47462-
you may not use this file except in compliance with the License.
47463-
You may obtain a copy of the License at
47464-
47465-
http://www.apache.org/licenses/LICENSE-2.0
47466-
47467-
Unless required by applicable law or agreed to in writing, software
47468-
distributed under the License is distributed on an "AS IS" BASIS,
47469-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
47470-
See the License for the specific language governing permissions and
47471-
limitations under the License.
47472-
47473-
---
47474-
47475-
Copyright (c) 2016 Mail.Ru Group
47476-
47477-
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
47478-
47479-
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
47480-
47481-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
47482-
47483-
4748447484
--------------------------------------------------------------------------------
4748547485
Dependency : go.etcd.io/bbolt
4748647486
Version: v1.3.6

‎go.mod‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/elastic/elastic-agent-shipper
33
go 1.17
44

55
require (
6-
github.com/elastic/elastic-agent-libs v0.2.16
6+
github.com/elastic/elastic-agent-libs v0.3.0
77
github.com/spf13/cobra v1.3.0
88
google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c // indirect
99
google.golang.org/grpc v1.48.0
@@ -14,7 +14,7 @@ require (
1414
github.com/eapache/go-resiliency v1.2.0
1515
github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220810153818-dd118efed5a5
1616
github.com/elastic/elastic-agent-client/v7 v7.0.1
17-
github.com/elastic/elastic-agent-shipper-client v0.4.1-0.20221028153110-a7eedbe6bd6c
17+
github.com/elastic/elastic-agent-shipper-client v0.5.0
1818
github.com/elastic/go-elasticsearch/v8 v8.2.0
1919
github.com/elastic/go-lumber v0.1.1
2020
github.com/elastic/go-ucfg v0.8.6 // indirect
@@ -80,7 +80,6 @@ require (
8080
go.elastic.co/apm/module/apmhttp/v2 v2.0.0 // indirect
8181
go.elastic.co/apm/v2 v2.1.0 // indirect
8282
go.elastic.co/ecszap v1.0.1 // indirect
83-
go.elastic.co/fastjson v1.1.0 // indirect
8483
go.uber.org/atomic v1.10.0 // indirect
8584
go.uber.org/multierr v1.9.0 // indirect
8685
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
@@ -98,7 +97,10 @@ require (
9897
howett.net/plist v1.0.0 // indirect
9998
)
10099

101-
require github.com/Shopify/sarama v1.27.0
100+
require (
101+
github.com/Shopify/sarama v1.27.0
102+
go.elastic.co/fastjson v1.1.0
103+
)
102104

103105
replace (
104106
github.com/Shopify/sarama => github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3

‎go.sum‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -502,11 +502,11 @@ github.com/elastic/elastic-agent-libs v0.2.2/go.mod h1:1xDLBhIqBIjhJ7lr2s+xRFFkQ
502502
github.com/elastic/elastic-agent-libs v0.2.5/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
503503
github.com/elastic/elastic-agent-libs v0.2.7/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
504504
github.com/elastic/elastic-agent-libs v0.2.9/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
505-
github.com/elastic/elastic-agent-libs v0.2.16 h1:31/VlnRp2ko0CMg+sm7b/u3vOPIlFXgMkJn/8uS6EbU=
506-
github.com/elastic/elastic-agent-libs v0.2.16/go.mod h1:0J9lzJh+BjttIiVjYDLncKYCEWUUHiiqnuI64y6C6ss=
505+
github.com/elastic/elastic-agent-libs v0.3.0 h1:cDq4aIP4sH+cMidlyU9M0POm1pfDE83RlG/Oh98tnC0=
506+
github.com/elastic/elastic-agent-libs v0.3.0/go.mod h1:GC2VyFWlgQzaG9OZiVpsA9r6Ff/jcvgtx+/afBvbjmI=
507507
github.com/elastic/elastic-agent-shipper-client v0.2.0/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q=
508-
github.com/elastic/elastic-agent-shipper-client v0.4.1-0.20221028153110-a7eedbe6bd6c h1:jgoz/UHtf6QunIwe8DmeQb0lq6y0jFsPO9rCA0l/cRY=
509-
github.com/elastic/elastic-agent-shipper-client v0.4.1-0.20221028153110-a7eedbe6bd6c/go.mod h1:OyI2W+Mv3JxlkEF3OeT7K0dbuxvwew8ke2Cf4HpLa9Q=
508+
github.com/elastic/elastic-agent-shipper-client v0.5.0 h1:rkdq7K8+ESNMXtMPzlwiiENTZz2Y6m4lN8SIMFrHuJA=
509+
github.com/elastic/elastic-agent-shipper-client v0.5.0/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI=
510510
github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
511511
github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE=
512512
github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
@@ -1158,6 +1158,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb
11581158
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
11591159
github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4=
11601160
github.com/mitchellh/hashstructure v0.0.0-20170116052023-ab25296c0f51/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
1161+
github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA=
11611162
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
11621163
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
11631164
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=

‎output/elasticsearch/output.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ package elasticsearch
77
import (
88
"bytes"
99
"context"
10-
"encoding/json"
10+
"fmt"
1111
"sync"
1212
"time"
1313

14+
"go.elastic.co/fastjson"
15+
1416
"github.com/elastic/elastic-agent-libs/logp"
1517
"github.com/elastic/elastic-agent-shipper-client/pkg/proto/messages"
1618
"github.com/elastic/elastic-agent-shipper/queue"
@@ -43,15 +45,19 @@ func NewElasticSearch(config *Config, reportCallback WatchReporter, queue *queue
4345
queue: queue,
4446
healthWatcher: newHealthWatcher(reportCallback, config.DegradedTimeout),
4547
}
46-
4748
return out
4849
}
4950

5051
func serializeEvent(event *messages.Event) ([]byte, error) {
5152
// TODO: we need to preprocessing the raw protobuf to get fields in the
5253
// right place for ECS. This just translates the protobuf structure
5354
// directly to json.
54-
return json.Marshal(event)
55+
jsonWriter := &fastjson.Writer{}
56+
err := fastjson.Marshal(jsonWriter, event.GetFields())
57+
if err != nil {
58+
return nil, fmt.Errorf("error marshalling event to json: %w", err)
59+
}
60+
return jsonWriter.Bytes(), nil
5561
}
5662

5763
// Start the elasticsearch output

‎output/elasticsearch/output_test.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License;
3+
// you may not use this file except in compliance with the Elastic License.
4+
5+
package elasticsearch
6+
7+
import (
8+
"encoding/json"
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
13+
"github.com/elastic/elastic-agent-libs/mapstr"
14+
"github.com/elastic/elastic-agent-shipper-client/pkg/helpers"
15+
"github.com/elastic/elastic-agent-shipper-client/pkg/proto/messages"
16+
)
17+
18+
func TestOutputMarshal(t *testing.T) {
19+
// elastic-agent-client has a more comprehensive test of the custom marshal, this is just to make sure we didn't break something
20+
testMessage, err := helpers.NewStruct(map[string]interface{}{
21+
"testvalue": "value",
22+
"data": 3,
23+
"map": mapstr.M{
24+
"value": "string",
25+
},
26+
})
27+
require.NoError(t, err)
28+
29+
evt := &messages.Event{Fields: testMessage}
30+
31+
jsonData, err := serializeEvent(evt)
32+
require.NoError(t, err)
33+
// unmarshal back to map
34+
evtMap := mapstr.M{}
35+
t.Logf("got: %s", string(jsonData))
36+
err = json.Unmarshal(jsonData, &evtMap)
37+
require.NoError(t, err)
38+
require.Equal(t, "value", evtMap["testvalue"])
39+
require.Equal(t, float64(3), evtMap["data"])
40+
require.Equal(t, map[string]interface{}{"value": "string"}, evtMap["map"])
41+
}

0 commit comments

Comments
 (0)