[Heartbeat] Setuid to regular user / lower capabilities when possible (elastic#27878)

andrewvc · web-flow · commit a78a980da2ef · 2021-10-12T19:37:20.000-05:00
partial fix for elastic#27648 , this PR: Detects if the user is running as root then: Checks to see if an environment variable BEAT_SETUID_AS (set in our Docker.tmpl) is present Attempts to Setuid , Setgid and Setgroups to that user / groups Invokes setcap to drop all privileges except NET_RAW+ep This PR also fixes the broken syscall filtering in heartbeat, some non-syscall strings were breaking that. With the changes here elastic-agent can still run as root, but the subprocesses can lower their privileges ASAP. This should also make it possible for heartbeat to safely run ICMP pings and synthetics. Synthetics must run as non-root, but ICMP requires NET_RAW. This lets us be consistent in our docs with the recommendation that elastic-agent run as root.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -325,6 +325,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
+- Fix broken seccomp filtering and improve security via `setcap` and `setuid` when running as root on linux in containers. {pull}27878[27878]
 
 *Journalbeat*
 
diff --git a/NOTICE.txt b/NOTICE.txt
diff --git a/dev-tools/notice/overrides.json b/dev-tools/notice/overrides.json
@@ -11,3 +11,5 @@
 {"name": "github.com/munnerz/goautoneg", "licenceType": "BSD-3-Clause"}
 {"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"}
 {"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"}
+{"name": "kernel.org/pub/linux/libs/security/libcap/cap", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
+{"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml
@@ -432,7 +432,7 @@ shared:
       dockerfile: 'Dockerfile.elastic-agent.tmpl'
       docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
       user: '{{ .BeatName }}'
-      linux_capabilities: ''
+      linux_capabilities: 'cap_net_raw+eip'
       image_name: ''
     files:
       'elastic-agent.yml':
diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
@@ -17,9 +17,6 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
     rm {{ $beatBinary }} && \
     ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \
     chmod 0755 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
-{{- if .linux_capabilities }}
-    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
-{{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
@@ -30,11 +27,20 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
 {{- end }}
     true
 
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
+{{- end }}
+
 FROM {{ .from }}
 
 # Contains the elastic agent image variant, an empty string for the standard variant
 # or "complete" for the bigger one.
 ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}}
+ENV BEAT_SETUID_AS={{ .user }}
 
 {{- if contains .from "ubi-minimal" }}
 RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq &&  microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl
@@ -13,14 +13,19 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
     find {{ $beatHome }} -type d -exec chmod 0755 {} \; && \
     find {{ $beatHome }} -type f -exec chmod 0644 {} \; && \
     chmod 0755 {{ $beatBinary }} && \
-{{- if .linux_capabilities }}
-    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
-{{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
     chmod 0775 {{ $beatHome }}/data {{ $beatHome }}/logs
 
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
+{{- end }}
+
 FROM {{ .from }}
 
 {{- if contains .from "ubi-minimal" }}
@@ -127,6 +132,7 @@ USER {{ .user }}
 {{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal")))  }}
 # Setup synthetics env vars
 ENV ELASTIC_SYNTHETICS_CAPABLE=true
+ENV BEAT_SETUID_AS={{ .user }}
 ENV SUITES_DIR={{ $beatHome }}/suites
 ENV NODE_VERSION=14.17.5
 ENV PATH="$NODE_PATH/node/bin:$PATH"
diff --git a/go.mod b/go.mod
@@ -192,6 +192,7 @@ require (
 	k8s.io/api v0.21.1
 	k8s.io/apimachinery v0.21.1
 	k8s.io/client-go v0.21.1
+	kernel.org/pub/linux/libs/security/libcap/cap v1.2.57
 )
 
 require (
@@ -283,6 +284,7 @@ require (
 	k8s.io/klog/v2 v2.8.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -1344,6 +1344,10 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 h1:2nmqI+aw7EQZuelYktkQHBE4jESD2tOR+lOJEnv/Apo=
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.57/go.mod h1:uI99C3r4SXvJeuqoEtx/eWt7UbmfqqZ80H8q+9t/A7I=
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 h1:NOFATXSf5z/cMR3HIwQ3Xrd3nwnWl5xThmNr5U/F0pI=
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.57/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go
@@ -20,6 +20,7 @@ package beater
 import (
 	"errors"
 	"fmt"
+	"syscall"
 	"time"
 
 	"github.com/elastic/beats/v7/heartbeat/config"
@@ -81,6 +82,9 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 func (bt *Heartbeat) Run(b *beat.Beat) error {
 	logp.Info("heartbeat is running! Hit CTRL-C to stop it.")
 
+	groups, _ := syscall.Getgroups()
+	logp.Info("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups)
+
 	stopStaticMonitors, err := bt.RunStaticMonitors(b)
 	if err != nil {
 		return err
diff --git a/heartbeat/scripts/mage/package.go b/heartbeat/scripts/mage/package.go
@@ -48,7 +48,7 @@ func CustomizePackaging() {
 		pkgType := args.Types[0]
 		switch pkgType {
 		case devtools.Docker:
-			args.Spec.ExtraVar("linux_capabilities", "cap_net_raw=eip")
+			args.Spec.ExtraVar("linux_capabilities", "cap_net_raw+eip")
 			args.Spec.Files[monitorsDTarget] = monitorsD
 		case devtools.TarGz, devtools.Zip:
 			args.Spec.Files[monitorsDTarget] = monitorsD
diff --git a/heartbeat/security.go b/heartbeat/security.go
@@ -0,0 +1,237 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// TODO: Support other architectures (e.g. arm)
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+	"syscall"
+
+	"kernel.org/pub/linux/libs/security/libcap/cap"
+
+	"github.com/elastic/beats/v7/libbeat/common/seccomp"
+)
+
+func init() {
+	// Here we set a bunch of linux specific security stuff.
+	// In the context of a container, where users frequently run as root, we follow BEAT_SETUID_AS to setuid/gid
+	// and add capabilities to make this actually run as a regular user. This also helps Node.js in synthetics, which
+	// does not want to run as root. It's also just generally more secure.
+	if localUserName := os.Getenv("BEAT_SETUID_AS"); localUserName != "" && syscall.Geteuid() == 0 {
+		err := changeUser(localUserName)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// Attempt to set capabilities before we setup seccomp rules
+	// Note that we discard any errors because they are not actionable.
+	// The beat should use `getcap` at a later point to examine available capabilities
+	// rather than relying on errors from `setcap`
+	setCapabilities()
+
+	switch runtime.GOARCH {
+	case "amd64", "386":
+		err := setSeccompRules()
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func changeUser(localUserName string) error {
+	localUser, err := user.Lookup(localUserName)
+	if err != nil {
+		return fmt.Errorf("could not lookup '%s': %w", localUser, err)
+	}
+	localUserUid, err := strconv.Atoi(localUser.Uid)
+	if err != nil {
+		return fmt.Errorf("could not parse UID '%s' as int: %w", localUser.Uid, err)
+	}
+	localUserGid, err := strconv.Atoi(localUser.Gid)
+	if err != nil {
+		return fmt.Errorf("could not parse GID '%s' as int: %w", localUser.Uid, err)
+	}
+	// We include the root group because the docker image contains many directories (data,logs)
+	// that are owned by root:root with 0775 perms. The heartbeat user is in both groups
+	// in the container, but we need to repeat that here.
+	err = syscall.Setgroups([]int{localUserGid, 0})
+	if err != nil {
+		return fmt.Errorf("could not set groups: %w", err)
+	}
+
+	// Set the main group as localUserUid so new files created are owned by the user's group
+	err = syscall.Setgid(localUserGid)
+	if err != nil {
+		return fmt.Errorf("could not set gid to %d: %w", localUserGid, err)
+	}
+
+	// Note this is not the regular SetUID! Look at the 'cap' package docs for it, it preserves
+	// capabilities post-SetUID, which we use to lock things down immediately
+	err = cap.SetUID(localUserUid)
+	if err != nil {
+		return fmt.Errorf("could not setuid to %d: %w", localUserUid, err)
+	}
+
+	// This may not be necessary, but is good hygeine, we do some shelling out to node/npm etc.
+	// and $HOME should reflect the user's preferences
+	return os.Setenv("HOME", localUser.HomeDir)
+}
+
+func setCapabilities() error {
+	// Start with an empty capability set
+	newcaps := cap.NewSet()
+	// Both permitted and effective are required! Permitted makes the permmission
+	// possible to get, effective makes it 'active'
+	err := newcaps.SetFlag(cap.Permitted, true, cap.NET_RAW)
+	if err != nil {
+		return fmt.Errorf("error setting permitted setcap: %w", err)
+	}
+	err = newcaps.SetFlag(cap.Effective, true, cap.NET_RAW)
+	if err != nil {
+		return fmt.Errorf("error setting effective setcap: %w", err)
+	}
+
+	// We do not want these capabilities to be inherited by subprocesses
+	err = newcaps.SetFlag(cap.Inheritable, false, cap.NET_RAW)
+	if err != nil {
+		return fmt.Errorf("error setting inheritable setcap: %w", err)
+	}
+
+	// Apply the new capabilities to the current process (incl. all threads)
+	err = newcaps.SetProc()
+	if err != nil {
+		return fmt.Errorf("error setting new process capabilities via setcap: %w", err)
+	}
+
+	return nil
+}
+
+func setSeccompRules() error {
+	// We require a number of syscalls to run. This list was generated with
+	// mage build && env ELASTIC_SYNTHETICS_CAPABLE=true strace -f --output=syscalls  ./heartbeat --path.config sample-synthetics-config/ -e
+	// then grepping for 'EPERM' in the 'syscalls' file.
+	syscalls := []string{
+		"access",
+		"arch_prctl",
+		"bind",
+		"brk",
+		"capget",
+		"chdir",
+		"chmod",
+		"chown",
+		"clone",
+		"close",
+		"connect",
+		"creat",
+		"dup2",
+		"epoll_ctl",
+		"epoll_pwait",
+		"eventfd2",
+		"execve",
+		"exit",
+		"faccessat",
+		"fadvise64",
+		"fallocate",
+		"fcntl",
+		"flock",
+		"fstat",
+		"fsync",
+		"futex",
+		"capget",
+		"getcwd",
+		"getdents64",
+		"getegid",
+		"geteuid",
+		"getgid",
+		"getpeername",
+		"getpgrp",
+		"getpid",
+		"getppid",
+		"getpriority",
+		"getrandom",
+		"getresuid",
+		"getresgid",
+		"getrusage",
+		"getsockname",
+		"gettid",
+		"getuid",
+		"ioctl",
+		"inotify_init",
+		"lchown",
+		"link",
+		"lseek",
+		"madvise",
+		"memfd_create",
+		"mkdir",
+		"mkdirat",
+		"mlock",
+		"mmap",
+		"mprotect",
+		"munmap",
+		"nanosleep",
+		"name_to_handle_at",
+		"newfstatat",
+		"openat",
+		"pipe",
+		"pipe2",
+		"poll",
+		"prctl",
+		"pread64",
+		"prlimit64",
+		"pwrite64",
+		"read",
+		"readlink",
+		"readlinkat",
+		"recvfrom",
+		"rename",
+		"rmdir",
+		"rt_sigaction",
+		"rt_sigprocmask",
+		"rt_sigreturn",
+		"sched_getaffinity",
+		"sched_getparam",
+		"sched_getscheduler",
+		"select",
+		"sendto",
+		"set_robust_list",
+		"set_tid_address",
+		"setpriority",
+		"setsid",
+		"sigaltstack",
+		"socket",
+		"socketpair",
+		"stat",
+		"statx",
+		"symlink",
+		"umask",
+		"uname",
+		"unlink",
+		"utimensat",
+		"write",
+	}
+
+	return seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...)
+}
diff --git a/packetbeat/scripts/mage/package.go b/packetbeat/scripts/mage/package.go
@@ -60,7 +60,7 @@ func CustomizePackaging() {
 				args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml)
 				args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml)
 			case devtools.Docker:
-				args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip")
+				args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin+eip")
 			default:
 				panic(errors.Errorf("unhandled package type: %v", pkgType))
 			}
diff --git a/x-pack/heartbeat/monitors/browser/browser.go b/x-pack/heartbeat/monitors/browser/browser.go
diff --git a/x-pack/heartbeat/monitors/browser/source/local.go b/x-pack/heartbeat/monitors/browser/source/local.go
diff --git a/x-pack/heartbeat/monitors/browser/source/zipurl.go b/x-pack/heartbeat/monitors/browser/source/zipurl.go
diff --git a/x-pack/heartbeat/seccomp_linux.go b/x-pack/heartbeat/seccomp_linux.go

Original file line number	Diff line number	Diff line change
`@@ -192,6 +192,7 @@ require (`
`192`	`192`	`k8s.io/api v0.21.1`
`193`	`193`	`k8s.io/apimachinery v0.21.1`
`194`	`194`	`k8s.io/client-go v0.21.1`
	`195`	`+ kernel.org/pub/linux/libs/security/libcap/cap v1.2.57`
`195`	`196`	`)`
`196`	`197`
`197`	`198`	`require (`
`@@ -283,6 +284,7 @@ require (`
`283`	`284`	`k8s.io/klog/v2 v2.8.0 // indirect`
`284`	`285`	`k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect`
`285`	`286`	`k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect`
	`287`	`+ kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 // indirect`
`286`	`288`	`sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect`
`287`	`289`	`sigs.k8s.io/yaml v1.2.0 // indirect`
`288`	`290`	`)`