Skip to content

Commit aa1775b

Browse files
gwossumgwossum
authored
feat: add TLS certificate reloading on SIGHUP (#27057)
Add TLS certificate reloading on SIGHUP. For httpd service certificates, the configuration is reloaded and certificate and key file locations are updated accordingly. For opentsdb service certificates, certificates and keys at the existing locations are reloaded. If reloading a certificate fails, the currently loaded certificate continues to be used. Also adds file permission checking for TLS certificates and private keys. Clean cherry-pick of #26994 to 1.12. Closes: #27056 (cherry picked from commit 8c9b850)
1 parent 1356a21 commit aa1775b

File tree

13 files changed

+2001
-70
lines changed

13 files changed

+2001
-70
lines changed

‎cmd/influxd/main.go‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,29 @@ func (m *Main) Run(args ...string) error {
8282
return fmt.Errorf("run: %s", err)
8383
}
8484

85-
signalCh := make(chan os.Signal, 1)
86-
signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
85+
shutdownCh := make(chan os.Signal, 1)
86+
signal.Notify(shutdownCh, os.Interrupt, syscall.SIGTERM)
87+
reloadCh := make(chan os.Signal, 1)
88+
signal.Notify(reloadCh, syscall.SIGHUP)
8789
cmd.Logger.Info("Listening for signals")
8890

89-
// Block until one of the signals above is received
90-
sig := <-signalCh
91+
logQueries := false
92+
shutdown := false
93+
for !shutdown {
94+
select {
95+
case sig := <-shutdownCh:
96+
if sig == syscall.SIGTERM {
97+
logQueries = true
98+
}
99+
shutdown = true
100+
101+
case <-reloadCh:
102+
cmd.ReloadConfig(args...)
103+
}
104+
}
105+
91106
cmd.Logger.Info("Signal received, initializing clean shutdown...")
92-
if sig == syscall.SIGTERM && cmd.Server.LogQueriesOnTermination() {
107+
if logQueries && cmd.Server.LogQueriesOnTermination() {
93108
cmd.Server.QueryExecutor.TaskManager.LogCurrentQueries(cmd.Logger.Info)
94109
}
95110
go cmd.Close()
@@ -98,7 +113,7 @@ func (m *Main) Run(args ...string) error {
98113
// or the Command is gracefully closed
99114
cmd.Logger.Info("Waiting for clean shutdown...")
100115
select {
101-
case <-signalCh:
116+
case <-shutdownCh:
102117
cmd.Logger.Info("Second signal received, initializing hard shutdown")
103118
case <-time.After(time.Second * 30):
104119
cmd.Logger.Info("Time limit reached, initializing hard shutdown")

‎cmd/influxd/run/command.go‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,27 +65,39 @@ func NewCommand() *Command {
6565
}
6666
}
6767

68-
// Run parses the config from args and runs the server.
69-
func (cmd *Command) Run(args ...string) error {
68+
func (cmd *Command) LoadConfig(args ...string) (Options, *Config, error) {
69+
fail := func(err error) (Options, *Config, error) { return Options{}, nil, err }
70+
7071
// Parse the command line flags.
7172
options, err := cmd.ParseFlags(args...)
7273
if err != nil {
73-
return err
74+
return fail(fmt.Errorf("error parsing command line: %w", err))
7475
}
7576

76-
config, err := cmd.ParseConfig(options.GetConfigPath())
77+
configPath := options.GetConfigPath()
78+
config, err := cmd.ParseConfig(configPath)
7779
if err != nil {
78-
return fmt.Errorf("parse config: %s", err)
80+
return fail(fmt.Errorf("error parsing config file (%q): %s", configPath, err))
7981
}
8082

8183
// Apply any environment variables on top of the parsed config
8284
if err := config.ApplyEnvOverrides(cmd.Getenv); err != nil {
83-
return fmt.Errorf("apply env config: %v", err)
85+
return fail(fmt.Errorf("error applying env config: %v", err))
8486
}
8587

8688
// Validate the configuration.
8789
if err := config.Validate(); err != nil {
88-
return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err)
90+
return fail(fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err))
91+
}
92+
93+
return options, config, nil
94+
}
95+
96+
// Run parses the config from args and runs the server.
97+
func (cmd *Command) Run(args ...string) error {
98+
options, config, err := cmd.LoadConfig(args...)
99+
if err != nil {
100+
return err
89101
}
90102

91103
var logErr error
@@ -179,6 +191,22 @@ func (cmd *Command) Close() error {
179191
return nil
180192
}
181193

194+
// ReloadConfig reloads the configuration and applies select configuration values to the running server.
195+
func (cmd *Command) ReloadConfig(args ...string) {
196+
log, logEnd := logger.NewOperation(cmd.Logger, "Reloading select configuration settings", "config_reload")
197+
defer logEnd()
198+
199+
_, reloadedConfig, err := cmd.LoadConfig(args...)
200+
if err != nil {
201+
log.Error("error reloading config", zap.Error(err))
202+
return
203+
}
204+
205+
if err := cmd.Server.ApplyReloadedConfig(reloadedConfig, log); err != nil {
206+
log.Error("error applying reloaded config; some config values may have been applied", zap.Error(err))
207+
}
208+
}
209+
182210
func (cmd *Command) monitorServerErrors() {
183211
logger := log.New(cmd.Stderr, "", log.LstdFlags)
184212
for {

‎cmd/influxd/run/server.go‎

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package run
33
import (
44
"context"
55
"crypto/tls"
6+
"errors"
67
"fmt"
78
"io"
89
"log"
@@ -87,10 +88,12 @@ type Server struct {
8788

8889
MetaClient *meta.Client
8990

90-
TSDBStore *tsdb.Store
91-
QueryExecutor *query.Executor
92-
PointsWriter *coordinator.PointsWriter
93-
Subscriber *subscriber.Service
91+
TSDBStore *tsdb.Store
92+
QueryExecutor *query.Executor
93+
PointsWriter *coordinator.PointsWriter
94+
Subscriber *subscriber.Service
95+
HttpdService *httpd.Service
96+
OpenTSDBServices []*opentsdb.Service
9497

9598
Services []Service
9699

@@ -343,6 +346,7 @@ func (s *Server) appendHTTPDService(c httpd.Config) error {
343346
s.Prometheus.MustRegister(srv.Handler.Controller.PrometheusCollectors()...)
344347
}
345348

349+
s.HttpdService = srv
346350
s.Services = append(s.Services, srv)
347351
return nil
348352
}
@@ -367,6 +371,8 @@ func (s *Server) appendOpenTSDBService(c opentsdb.Config) error {
367371
}
368372
srv.PointsWriter = s.PointsWriter
369373
srv.MetaClient = s.MetaClient
374+
375+
s.OpenTSDBServices = append(s.OpenTSDBServices, srv)
370376
s.Services = append(s.Services, srv)
371377
return nil
372378
}
@@ -574,6 +580,52 @@ func (s *Server) Close() error {
574580
return nil
575581
}
576582

583+
func (s *Server) ApplyReloadedConfig(config *Config, log *zap.Logger) error {
584+
if log == nil {
585+
log = s.Logger
586+
}
587+
588+
// Verify reloading each service's config and gather the apply functions.
589+
// During the verify stage, we will abort if any service's verify returns an error.
590+
var applyFuncs []func() error
591+
592+
if s.HttpdService != nil {
593+
if af, err := s.HttpdService.PrepareReloadConfig(config.HTTPD); err == nil {
594+
applyFuncs = append(applyFuncs, af)
595+
} else {
596+
log.Error("error reloading httpd service config, no new configuration applied", zap.Error(err))
597+
return err
598+
}
599+
600+
}
601+
602+
// Because we don't have a way of matching OpenTSDBInput configurations with
603+
// the OpenTSDB services they created, we will just reload the TLS certificate
604+
// at the currently configured paths.
605+
for _, srv := range s.OpenTSDBServices {
606+
if af, err := srv.PrepareReloadTLSCertificates(); err == nil {
607+
applyFuncs = append(applyFuncs, af)
608+
} else {
609+
log.Error("error reloading OpenTSDB service TLS certificate, no new configuration applied", zap.Error(err),
610+
zap.String("bind-address", srv.BindAddress),
611+
zap.String("database", srv.Database), zap.String("retention-policy", srv.RetentionPolicy))
612+
return err
613+
}
614+
}
615+
616+
// We've verified that the configuration should load, now apply it. Keep going even if something fails to apply.
617+
var applyErrs []error
618+
for _, af := range applyFuncs {
619+
if af != nil {
620+
if err := af(); err != nil {
621+
log.Error("error applying configuration reload, continuing apply", zap.Error(err))
622+
applyErrs = append(applyErrs, err)
623+
}
624+
}
625+
}
626+
return errors.Join(applyErrs...)
627+
}
628+
577629
// startServerReporting starts periodic server reporting.
578630
func (s *Server) startServerReporting() {
579631
s.reportServer()

‎pkg/file/file.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
package file
22

33
import (
4+
goerrors "errors"
45
"fmt"
56
"io"
67
"os"
78

89
"github.com/influxdata/influxdb/pkg/errors"
910
)
1011

12+
var (
13+
ErrPermissionsTooOpen = goerrors.New("file permissions are too open")
14+
ErrNilParam = goerrors.New("got nil parameter")
15+
)
16+
1117
func copyFile(src, dst string) (err error) {
1218
in, err := os.Open(src)
1319
if err != nil {
@@ -42,3 +48,41 @@ func MoveFileWithReplacement(src, dst string) error {
4248

4349
return os.Remove(src)
4450
}
51+
52+
// VerifyFilePermissivenessF checks if permissions on f are as restrictive
53+
// or more restrictive than maxPerms. if not, then an error is returned.
54+
// For security reasons, there is no VerifyFilePermissiveness function
55+
// that allows passing a path. This is to prevent TOCTOU (Time-of-Check-Time-of-Use)
56+
// issues because of race conditions on checking file permissions versus
57+
// opening the file.
58+
func VerifyFilePermissivenessF(f *os.File, maxPerms os.FileMode) error {
59+
if f == nil {
60+
return fmt.Errorf("VerifyFilePermissivenessF: %w", ErrNilParam)
61+
}
62+
63+
info, err := f.Stat()
64+
if err != nil {
65+
return fmt.Errorf("stat failed for %q: %w", f.Name(), err)
66+
}
67+
return VerifyFileInfoPermissiveness(info, maxPerms, f.Name())
68+
}
69+
70+
// VerifyFileInfoPermissiveness checks if permissions on info are as restrictive
71+
// or more restrictive than maxPerms. If not, then an error is returned. path is
72+
// only used to provide better messages. If path is empty, then info.Name() is used
73+
// for error messages.
74+
func VerifyFileInfoPermissiveness(info os.FileInfo, maxPerms os.FileMode, path string) error {
75+
if info == nil {
76+
return fmt.Errorf("VerifyFileInfoPermissiveness: %w", ErrNilParam)
77+
}
78+
perms := info.Mode().Perm()
79+
extraPerms := perms & ^maxPerms
80+
if extraPerms != 0 {
81+
if len(path) == 0 {
82+
path = info.Name()
83+
}
84+
return fmt.Errorf("%w: for %q, maximum is %04o (%s) but found %04o (%s); extra permissions: %04o (%s)",
85+
ErrPermissionsTooOpen, path, maxPerms, maxPerms, perms, perms, extraPerms, extraPerms)
86+
}
87+
return nil
88+
}

0 commit comments

Comments
 (0)