Skip to content

Commit aead51b

Browse files
aleksmausaleksmaus
authored
Adjust integration tests to make them work with fleet service token. (#1039)
* Adjust integration tests to make them work with fleet service token. * Remove .fleet-* mapping cruft. Test rely on .fleet indices plugin to bootstrap indices/datastreams appropriately. * Update integration tests setup to use the token for setup
1 parent 35555cf commit aead51b

28 files changed

Lines changed: 210 additions & 608 deletions

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,6 @@ fleet-server.dev.yml
1212
*.log
1313
*.log.*
1414

15-
dev-tools/integration/.env.bck
15+
dev-tools/integration/.env.bck
16+
17+
.service_token

‎Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ local: ## - Build local binary for local environment (bin/fleet-server)
4949
clean: ## - Clean up build artifacts
5050
@printf "${CMD_COLOR_ON} Clean up build artifacts\n${CMD_COLOR_OFF}"
5151
rm -rf ./bin/ ./build/
52+
rm .service_token
5253

5354
.PHONY: generate
5455
generate: ## - Generate schema models
@@ -214,4 +215,6 @@ test-int: prepare-test-context ## - Run integration tests with full setup (slow
214215
.PHONY: test-int-set
215216
test-int-set: ## - Run integration tests without setup
216217
# Initialize indices one before running all the tests
217-
ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}) ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} go test -v -tags=integration -count=1 -race ./...
218+
ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_ELASTICSEARCH_HOSTS}) \
219+
ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} \
220+
go test -v -tags=integration -count=1 -race -p 1 ./...

‎NOTICE.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
Elastic Beats
2-
Copyright 2014-2021 Elasticsearch BV
2+
Copyright 2014-2022 Elasticsearch BV
33

44
This product includes software developed by The Apache Software
55
Foundation (http://www.apache.org/).

‎dev-tools/integration/get-elasticsearch-servicetoken.sh‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,19 @@
11
#!/bin/bash
22

3-
set -e
4-
53
host="$1"
64

7-
jsonBody="$(curl -fsSL -XPOST "$host/_security/service/elastic/fleet-server/credential/token/token1")"
5+
jsonBody="$(curl -sSL -XPOST "$host/_security/service/elastic/fleet-server/credential/token/token1")"
6+
87
# use grep and sed to get the service token value as we may not have jq or a similar tool on the instance
98
token=$(echo ${jsonBody} | grep -Eo '"value"[^}]*' | grep -Eo ':.*' | sed -r "s/://" | sed -r 's/"//g')
9+
10+
# cache or use cached token in order to be able to run repeative integration tests,
11+
# very useful during development, without recreating elasticsearch instance every time.
12+
if [ -z "$token" ]
13+
then
14+
token=`cat .service_token`
15+
else
16+
echo "$token" > .service_token
17+
fi
18+
1019
echo $token

‎internal/pkg/bulk/bulk_integration_test.go‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
)
2323

2424
func TestBulkCreate(t *testing.T) {
25-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
2625
ctx, cn := context.WithCancel(context.Background())
2726
defer cn()
2827

@@ -125,7 +124,6 @@ func TestBulkCreate(t *testing.T) {
125124
}
126125

127126
func TestBulkCreateBody(t *testing.T) {
128-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
129127
ctx, cn := context.WithCancel(context.Background())
130128
defer cn()
131129

@@ -179,7 +177,6 @@ func TestBulkCreateBody(t *testing.T) {
179177
}
180178

181179
func TestBulkIndex(t *testing.T) {
182-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
183180
ctx, cn := context.WithCancel(context.Background())
184181
defer cn()
185182

@@ -203,7 +200,6 @@ func TestBulkIndex(t *testing.T) {
203200
}
204201

205202
func TestBulkUpdate(t *testing.T) {
206-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
207203
ctx, cn := context.WithCancel(context.Background())
208204
defer cn()
209205

@@ -242,7 +238,6 @@ func TestBulkUpdate(t *testing.T) {
242238
}
243239

244240
func TestBulkSearch(t *testing.T) {
245-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
246241
ctx, cn := context.WithCancel(context.Background())
247242
defer cn()
248243

@@ -285,7 +280,6 @@ func TestBulkSearch(t *testing.T) {
285280
}
286281

287282
func TestBulkDelete(t *testing.T) {
288-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
289283
ctx, cn := context.WithCancel(context.Background())
290284
defer cn()
291285

‎internal/pkg/bulk/setup_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ import (
1414

1515
"github.com/elastic/go-ucfg/yaml"
1616
"github.com/rs/xid"
17+
"github.com/rs/zerolog"
1718

1819
"github.com/Pallinder/go-randomdata"
1920
"github.com/elastic/fleet-server/v7/internal/pkg/config"
2021
"github.com/elastic/fleet-server/v7/internal/pkg/es"
2122
"github.com/elastic/fleet-server/v7/internal/pkg/testing/esutil"
22-
"github.com/rs/zerolog"
23+
"github.com/elastic/fleet-server/v7/internal/pkg/testutil"
2324
)
2425

2526
var defaultCfg config.Config
@@ -115,7 +116,9 @@ func init() {
115116
func SetupBulk(ctx context.Context, t testing.TB, opts ...BulkOpt) Bulk {
116117
t.Helper()
117118

118-
cli, err := es.NewClient(ctx, &defaultCfg, false)
119+
// Set up the client with username and password since this test is generic for any index and uses it's own index/mapping
120+
e := testutil.GetEnvironment()
121+
cli, err := es.NewClient(ctx, &defaultCfg, false, es.WithUsrPwd(e.Username, e.Password))
119122
if err != nil {
120123
t.Fatal(err)
121124
}

‎internal/pkg/coordinator/monitor_integration_test.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,12 @@ import (
2323
"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
2424
"github.com/elastic/fleet-server/v7/internal/pkg/config"
2525
"github.com/elastic/fleet-server/v7/internal/pkg/dl"
26-
"github.com/elastic/fleet-server/v7/internal/pkg/es"
2726
"github.com/elastic/fleet-server/v7/internal/pkg/model"
2827
"github.com/elastic/fleet-server/v7/internal/pkg/monitor"
2928
ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
3029
)
3130

3231
func TestMonitorLeadership(t *testing.T) {
33-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
3432
parentCtx := context.Background()
3533
bulkCtx, bulkCn := context.WithCancel(parentCtx)
3634
defer bulkCn()
@@ -39,9 +37,11 @@ func TestMonitorLeadership(t *testing.T) {
3937

4038
// flush bulker on every operation
4139
bulker := ftesting.SetupBulk(bulkCtx, t, bulk.WithFlushThresholdCount(1))
42-
serversIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingServer)
43-
policiesIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingPolicy)
44-
leadersIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingPolicyLeader)
40+
41+
serversIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetServers)
42+
policiesIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetPolicies)
43+
leadersIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetPoliciesLeader)
44+
4545
pim, err := monitor.New(policiesIndex, bulker.Client(), bulker.Client())
4646
if err != nil {
4747
t.Fatal(err)
@@ -115,7 +115,6 @@ func TestMonitorLeadership(t *testing.T) {
115115
}
116116

117117
func TestMonitorUnenroller(t *testing.T) {
118-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
119118
parentCtx := context.Background()
120119
bulkCtx, bulkCn := context.WithCancel(parentCtx)
121120
defer bulkCn()
@@ -124,10 +123,12 @@ func TestMonitorUnenroller(t *testing.T) {
124123

125124
// flush bulker on every operation
126125
bulker := ftesting.SetupBulk(bulkCtx, t, bulk.WithFlushThresholdCount(1))
127-
serversIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingServer)
128-
policiesIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingPolicy)
129-
leadersIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingPolicyLeader)
130-
agentsIndex := ftesting.SetupIndex(bulkCtx, t, bulker, es.MappingAgent)
126+
127+
serversIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetServers)
128+
policiesIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetPolicies)
129+
leadersIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetPoliciesLeader)
130+
agentsIndex := ftesting.CleanIndex(ctx, t, bulker, dl.FleetAgents)
131+
131132
pim, err := monitor.New(policiesIndex, bulker.Client(), bulker.Client())
132133
require.NoError(t, err)
133134
cfg := makeFleetConfig()

‎internal/pkg/dl/action_results_integration_test.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import (
1818
"github.com/rs/xid"
1919

2020
"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
21-
"github.com/elastic/fleet-server/v7/internal/pkg/es"
2221
"github.com/elastic/fleet-server/v7/internal/pkg/model"
2322
"github.com/elastic/fleet-server/v7/internal/pkg/rnd"
2423
ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
@@ -79,7 +78,8 @@ func storeRandomActionResults(ctx context.Context, bulker bulk.Bulk, index strin
7978
}
8079

8180
func setupActionResults(ctx context.Context, t *testing.T) (string, bulk.Bulk, []model.ActionResult) {
82-
index, bulker := ftesting.SetupIndexWithBulk(ctx, t, es.MappingActionResult)
81+
index, bulker := ftesting.SetupCleanIndex(ctx, t, FleetEnrollmentAPIKeys)
82+
8383
results, err := storeRandomActionResults(ctx, bulker, index)
8484
if err != nil {
8585
t.Fatal(err)
@@ -99,8 +99,7 @@ func (acrs ActionsResults) find(ar model.ActionResult) *model.ActionResult {
9999
return nil
100100
}
101101

102-
func TestActionResultsStored(t *testing.T) {
103-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
102+
func TestActionsResultsStored(t *testing.T) {
104103
ctx, cn := context.WithCancel(context.Background())
105104
defer cn()
106105

‎internal/pkg/dl/actions_integration_test.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,31 @@ import (
1414

1515
"github.com/google/go-cmp/cmp"
1616

17+
"github.com/elastic/fleet-server/v7/internal/pkg/gcheckpt"
1718
ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
1819
)
1920

2021
func TestSearchActionsQuery(t *testing.T) {
21-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
2222
ctx, cn := context.WithCancel(context.Background())
2323
defer cn()
2424

2525
now := time.Now().UTC()
2626

2727
index, bulker, actions := ftesting.SetupActions(ctx, t, 1, 11)
2828

29+
checkpoint, err := gcheckpt.Query(ctx, bulker.Client(), index)
30+
if err != nil {
31+
t.Fatal(err)
32+
}
33+
34+
maxSeqNo := checkpoint.Value()
35+
minSeqNo := maxSeqNo - int64(len(actions))
36+
2937
t.Run("all agents actions", func(t *testing.T) {
3038

3139
foundActions, err := findActions(ctx, bulker, QueryAllAgentActions, index, map[string]interface{}{
32-
FieldSeqNo: -1,
33-
FieldMaxSeqNo: len(actions),
40+
FieldSeqNo: minSeqNo,
41+
FieldMaxSeqNo: maxSeqNo,
3442
FieldExpiration: now,
3543
}, nil)
3644
if err != nil {

‎internal/pkg/dl/agent_integration_test.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,15 @@ import (
1818
"github.com/stretchr/testify/require"
1919

2020
"github.com/elastic/fleet-server/v7/internal/pkg/bulk"
21-
"github.com/elastic/fleet-server/v7/internal/pkg/es"
2221
"github.com/elastic/fleet-server/v7/internal/pkg/model"
2322
ftesting "github.com/elastic/fleet-server/v7/internal/pkg/testing"
2423
)
2524

2625
func TestFindOfflineAgents(t *testing.T) {
27-
t.Skip("Skipping broken integration test as template creation does not work with a service token.")
2826
ctx, cn := context.WithCancel(context.Background())
2927
defer cn()
3028

31-
index, bulker := ftesting.SetupIndexWithBulk(ctx, t, es.MappingAgent)
29+
index, bulker := ftesting.SetupCleanIndex(ctx, t, FleetAgents)
3230

3331
now := time.Now().UTC()
3432
nowStr := now.Format(time.RFC3339)

0 commit comments

Comments
 (0)