Skip to content

Commit 0dcf688

Browse files
axwmergify-bot
axw
authored and
mergify-bot
committed
systemtest: fix apm-server binary injection (#5440)
Due to some changes in elastic-agent (elastic/beats#24817), injection of the apm-server binary became ineffective and we have been running system tests with the published artifacts. Artifacts (such as the apm-server) are now unpacked into state/data/install/<artifact>. The state/data/install directory is expected to be owned by the elastic-agent user, so we can no longer bind mount the apm-server binary. Instead, we now create a custom Docker image and copy in the apm-server and apm-server.yml files. (cherry picked from commit 301caed)
1 parent c312bd8 commit 0dcf688

2 files changed

Lines changed: 111 additions & 67 deletions

File tree

‎systemtest/containers.go‎

Lines changed: 111 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
package systemtest
1919

2020
import (
21+
"archive/tar"
22+
"bytes"
2123
"context"
2224
"encoding/json"
2325
"errors"
@@ -29,8 +31,8 @@ import (
2931
"net/url"
3032
"os"
3133
"os/exec"
32-
"path"
3334
"path/filepath"
35+
"runtime"
3436
"strings"
3537
"time"
3638

@@ -42,6 +44,7 @@ import (
4244
"github.com/testcontainers/testcontainers-go/wait"
4345
"golang.org/x/sync/errgroup"
4446

47+
"github.com/elastic/apm-server/systemtest/apmservertest"
4548
"github.com/elastic/apm-server/systemtest/estest"
4649
"github.com/elastic/go-elasticsearch/v7"
4750
)
@@ -281,6 +284,11 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
281284
Scheme: "https",
282285
Host: net.JoinHostPort(fleetServerIPAddress, fleetServerPort),
283286
}
287+
containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
288+
hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem")
289+
if err != nil {
290+
return nil, err
291+
}
284292

285293
// Use the same stack version as used for fleet-server.
286294
agentImageVersion := fleetServerContainer.Image[strings.LastIndex(fleetServerContainer.Image, ":")+1:]
@@ -292,43 +300,34 @@ func NewUnstartedElasticAgentContainer() (*ElasticAgentContainer, error) {
292300
if err != nil {
293301
return nil, err
294302
}
295-
agentVCSRef := agentImageDetails.Config.Labels["org.label-schema.vcs-ref"]
296-
agentDataHashDir := path.Join("/usr/share/elastic-agent/data", "elastic-agent-"+agentVCSRef[:6])
297-
agentInstallDir := path.Join(agentDataHashDir, "install")
303+
stackVersion := agentImageDetails.Config.Labels["org.label-schema.version"]
304+
305+
// Build a custom elastic-agent image with a locally built apm-server binary injected.
306+
agentImage, err = buildElasticAgentImage(context.Background(), docker, stackVersion, agentImageVersion)
307+
if err != nil {
308+
return nil, err
309+
}
298310

299311
req := testcontainers.ContainerRequest{
300312
Image: agentImage,
301313
AutoRemove: true,
302314
Networks: networks,
315+
BindMounts: map[string]string{hostCACertPath: containerCACertPath},
303316
Env: map[string]string{
304-
// NOTE(axw) because we bind-mount the apm-server artifacts in, they end up owned by the
305-
// current user rather than root. Disable Beats's strict permission checks to avoid resulting
306-
// complaints, as they're irrelevant to these system tests.
307-
"BEAT_STRICT_PERMS": "false",
317+
"FLEET_URL": fleetServerURL.String(),
318+
"FLEET_CA": containerCACertPath,
308319
},
309320
}
310321
return &ElasticAgentContainer{
311-
request: req,
312-
installDir: agentInstallDir,
313-
fleetServerURL: fleetServerURL.String(),
314-
StackVersion: agentImageVersion,
315-
BindMountInstall: make(map[string]string),
322+
request: req,
323+
StackVersion: agentImageVersion,
316324
}, nil
317325
}
318326

319327
// ElasticAgentContainer represents an ephemeral Elastic Agent container.
320328
type ElasticAgentContainer struct {
321-
container testcontainers.Container
322-
request testcontainers.ContainerRequest
323-
fleetServerURL string
324-
325-
// installDir holds the location of the "install" directory inside
326-
// the Elastic Agent container.
327-
//
328-
// This will be set when the ElasticAgentContainer object is created,
329-
// and can be used to anticipate the location into which artifacts
330-
// can be bind-mounted.
331-
installDir string
329+
container testcontainers.Container
330+
request testcontainers.ContainerRequest
332331

333332
// StackVersion holds the stack version of the container image,
334333
// e.g. 8.0.0-SNAPSHOT.
@@ -344,11 +343,6 @@ type ElasticAgentContainer struct {
344343
// by exposed port. This will be populated by Start.
345344
Addrs map[string]string
346345

347-
// BindMountInstall holds a map of files to bind mount into the
348-
// container, mapping from the host location to target paths relative
349-
// to the install directory in the container.
350-
BindMountInstall map[string]string
351-
352346
// FleetEnrollmentToken holds an optional Fleet enrollment token to
353347
// use for enrolling the agent with Fleet. The agent will only enroll
354348
// if this is specified.
@@ -366,27 +360,12 @@ func (c *ElasticAgentContainer) Start() error {
366360
defer cancel()
367361

368362
// Update request from user-definable fields.
369-
c.request.Env["FLEET_URL"] = c.fleetServerURL
370363
if c.FleetEnrollmentToken != "" {
371364
c.request.Env["FLEET_ENROLL"] = "1"
372365
c.request.Env["FLEET_ENROLLMENT_TOKEN"] = c.FleetEnrollmentToken
373366
}
374-
375367
c.request.ExposedPorts = c.ExposedPorts
376368
c.request.WaitingFor = c.WaitingFor
377-
c.request.BindMounts = map[string]string{}
378-
for source, target := range c.BindMountInstall {
379-
c.request.BindMounts[source] = path.Join(c.installDir, target)
380-
}
381-
382-
// Inject CA certificate for verifying fleet-server.
383-
containerCACertPath := "/etc/pki/tls/certs/fleet-ca.pem"
384-
hostCACertPath, err := filepath.Abs("../testing/docker/fleet-server/ca.pem")
385-
if err != nil {
386-
return err
387-
}
388-
c.request.BindMounts[hostCACertPath] = containerCACertPath
389-
c.request.Env["FLEET_CA"] = containerCACertPath
390369

391370
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
392371
ContainerRequest: c.request,
@@ -457,3 +436,91 @@ func matchFleetServerAPIStatusHealthy(r io.Reader) bool {
457436
}
458437
return status.Status == "HEALTHY"
459438
}
439+
440+
// buildElasticAgentImage builds a Docker image from the published image with a locally built apm-server injected.
441+
func buildElasticAgentImage(ctx context.Context, docker *client.Client, stackVersion, imageVersion string) (string, error) {
442+
imageName := fmt.Sprintf("elastic-agent-systemtest:%s", imageVersion)
443+
log.Printf("Building image %s...", imageName)
444+
445+
// Build apm-server, and copy it into the elastic-agent container's "install" directory.
446+
// This bypasses downloading the artifact.
447+
arch := runtime.GOARCH
448+
if arch == "amd64" {
449+
arch = "x86_64"
450+
}
451+
apmServerInstallDir := fmt.Sprintf("./state/data/install/apm-server-%s-linux-%s", stackVersion, arch)
452+
apmServerBinary, err := apmservertest.BuildServerBinary("linux")
453+
if err != nil {
454+
return "", err
455+
}
456+
457+
// Binaries to copy from disk into the build context.
458+
binaries := map[string]string{
459+
"apm-server": apmServerBinary,
460+
}
461+
462+
// Generate Dockerfile contents.
463+
var dockerfile bytes.Buffer
464+
fmt.Fprintf(&dockerfile, "FROM docker.elastic.co/beats/elastic-agent:%s\n", imageVersion)
465+
fmt.Fprintf(&dockerfile, "COPY --chown=elastic-agent:elastic-agent apm-server apm-server.yml %s/\n", apmServerInstallDir)
466+
467+
// Files to generate in the build context.
468+
generatedFiles := map[string][]byte{
469+
"Dockerfile": dockerfile.Bytes(),
470+
"apm-server.yml": []byte(""),
471+
}
472+
473+
var buildContext bytes.Buffer
474+
tarw := tar.NewWriter(&buildContext)
475+
for name, path := range binaries {
476+
f, err := os.Open(path)
477+
if err != nil {
478+
return "", err
479+
}
480+
defer f.Close()
481+
info, err := f.Stat()
482+
if err != nil {
483+
return "", err
484+
}
485+
if err := tarw.WriteHeader(&tar.Header{
486+
Name: name,
487+
Size: info.Size(),
488+
Mode: 0755,
489+
Uname: "elastic-agent",
490+
Gname: "elastic-agent",
491+
}); err != nil {
492+
return "", err
493+
}
494+
if _, err := io.Copy(tarw, f); err != nil {
495+
return "", err
496+
}
497+
}
498+
for name, content := range generatedFiles {
499+
if err := tarw.WriteHeader(&tar.Header{
500+
Name: name,
501+
Size: int64(len(content)),
502+
Mode: 0644,
503+
Uname: "elastic-agent",
504+
Gname: "elastic-agent",
505+
}); err != nil {
506+
return "", err
507+
}
508+
if _, err := tarw.Write(content); err != nil {
509+
return "", err
510+
}
511+
}
512+
if err := tarw.Close(); err != nil {
513+
return "", err
514+
}
515+
516+
resp, err := docker.ImageBuild(ctx, &buildContext, types.ImageBuildOptions{Tags: []string{imageName}})
517+
if err != nil {
518+
return "", err
519+
}
520+
defer resp.Body.Close()
521+
if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
522+
return "", err
523+
}
524+
log.Printf("Built image %s", imageName)
525+
return imageName, nil
526+
}

‎systemtest/fleet_test.go‎

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,8 @@ package systemtest_test
1919

2020
import (
2121
"context"
22-
"fmt"
2322
"io/ioutil"
2423
"net/url"
25-
"path"
26-
"path/filepath"
27-
"runtime"
2824
"testing"
2925
"time"
3026

@@ -35,7 +31,6 @@ import (
3531
"go.elastic.co/apm/transport"
3632

3733
"github.com/elastic/apm-server/systemtest"
38-
"github.com/elastic/apm-server/systemtest/apmservertest"
3934
"github.com/elastic/apm-server/systemtest/fleettest"
4035
)
4136

@@ -80,24 +75,6 @@ func TestFleetIntegration(t *testing.T) {
8075
}
8176
}()
8277

83-
// Build apm-server, and bind-mount it into the elastic-agent container's "install"
84-
// directory. This bypasses downloading the artifact.
85-
arch := runtime.GOARCH
86-
if arch == "amd64" {
87-
arch = "x86_64"
88-
}
89-
apmServerArtifactName := fmt.Sprintf("apm-server-%s-linux-%s", agent.StackVersion, arch)
90-
91-
// Bind-mount the apm-server binary and apm-server.yml into the container's
92-
// "install" directory. This causes elastic-agent to skip installing the
93-
// artifact.
94-
apmServerBinary, err := apmservertest.BuildServerBinary("linux")
95-
require.NoError(t, err)
96-
agent.BindMountInstall[apmServerBinary] = path.Join(apmServerArtifactName, "apm-server")
97-
apmServerConfigFile, err := filepath.Abs("../apm-server.yml")
98-
require.NoError(t, err)
99-
agent.BindMountInstall[apmServerConfigFile] = path.Join(apmServerArtifactName, "apm-server.yml")
100-
10178
// Start elastic-agent with port 8200 exposed, and wait for the server to service
10279
// healthcheck requests to port 8200.
10380
agent.ExposedPorts = []string{"8200"}

0 commit comments

Comments
 (0)