Skip to content

Commit a913f10

Browse files
maaarcelinomaaarcelino
authored
feat: Redact env secrets from output and logs (#20691)
This adds functionality to redact secrets (aka environment variables) in cloud sync environments. This redacts them in the logs and in the output of all the CLI commands.
1 parent 50f909e commit a913f10

6 files changed

Lines changed: 128 additions & 10 deletions

File tree

‎cli/cmd/logging.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"os"
77

88
"github.com/cloudquery/cloudquery/cli/v6/internal/enum"
9+
"github.com/cloudquery/cloudquery/cli/v6/internal/secrets"
910
"github.com/rs/zerolog"
1011
"github.com/rs/zerolog/log"
1112
)
@@ -48,6 +49,7 @@ func initLogging(noLogFile bool, logLevel *enum.Enum, logFormat *enum.Enum, logC
4849
}
4950
}
5051
mw := io.MultiWriter(writers...)
51-
log.Logger = zerolog.New(mw).Level(zerologLevel).With().Str("module", "cli").Str("invocation_id", invocationUUID.String()).Timestamp().Logger()
52+
secretAwareWriter := secrets.NewSecretAwareWriter(mw, secretAwareRedactor)
53+
log.Logger = zerolog.New(secretAwareWriter).Level(zerologLevel).With().Str("module", "cli").Str("invocation_id", invocationUUID.String()).Timestamp().Logger()
5254
return logFile, nil
5355
}

‎cli/cmd/root.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ import (
55
"os"
66
"time"
77

8-
analytics "github.com/cloudquery/cloudquery/cli/v6/internal/analytics"
8+
"github.com/cloudquery/cloudquery/cli/v6/internal/analytics"
99
"github.com/cloudquery/cloudquery/cli/v6/internal/enum"
1010
"github.com/cloudquery/cloudquery/cli/v6/internal/env"
11+
"github.com/cloudquery/cloudquery/cli/v6/internal/secrets"
1112
"github.com/cloudquery/cloudquery/cli/v6/internal/uuid"
1213
guuid "github.com/google/uuid"
1314
"github.com/rs/zerolog"
@@ -28,11 +29,12 @@ High performance data integration at scale.
2829
Find more information at:
2930
https://www.cloudquery.io`
3031

31-
disableSentry = false
32-
logConsole = false
33-
oldAnalyticsClient *AnalyticsClient
34-
logFile *os.File
35-
invocationUUID uuid.UUID
32+
disableSentry = false
33+
logConsole = false
34+
oldAnalyticsClient *AnalyticsClient
35+
logFile *os.File
36+
invocationUUID uuid.UUID
37+
secretAwareRedactor *secrets.SecretAwareRedactor
3638
)
3739

3840
func NewCmdRoot() *cobra.Command {
@@ -47,6 +49,7 @@ func NewCmdRoot() *cobra.Command {
4749
fmt.Fprintf(os.Stderr, "failed to generate invocation uuid: %v", err)
4850
os.Exit(1)
4951
}
52+
secretAwareRedactor = secrets.NewSecretAwareRedactor()
5053

5154
// support legacy telemetry environment variable,
5255
// but the newer CQ_TELEMETRY_LEVEL environment variable takes precedence
@@ -109,6 +112,9 @@ func NewCmdRoot() *cobra.Command {
109112
},
110113
}
111114

115+
cmd.SetErr(secrets.NewSecretAwareWriter(os.Stderr, secretAwareRedactor))
116+
cmd.SetOut(secrets.NewSecretAwareWriter(os.Stdout, secretAwareRedactor))
117+
112118
cmd.PersistentFlags().String("cq-dir", ".cq", "directory to store cloudquery files, such as downloaded plugins")
113119
cmd.PersistentFlags().String("data-dir", "", "set persistent data directory")
114120
err = cmd.PersistentFlags().MarkDeprecated("data-dir", "use cq-dir instead")

‎cli/cmd/sync.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,7 @@ func sync(cmd *cobra.Command, args []string) error {
248248
}
249249
if isolatePluginEnvironment {
250250
cfg.Environment = filterPluginEnv(osEnviron, source.Name, "source")
251+
secretAwareRedactor.AddSecretEnv(cfg.Environment)
251252
}
252253
sourcePluginClient, err := managedplugin.NewClient(ctx, managedplugin.PluginSource, cfg, opts...)
253254
if err != nil {
@@ -291,6 +292,7 @@ func sync(cmd *cobra.Command, args []string) error {
291292
}
292293
if isolatePluginEnvironment {
293294
cfg.Environment = filterPluginEnv(osEnviron, destination.Name, "destination")
295+
secretAwareRedactor.AddSecretEnv(cfg.Environment)
294296
}
295297
destPluginClient, err := managedplugin.NewClient(ctx, managedplugin.PluginDestination, cfg, opts...)
296298
if err != nil {

‎cli/cmd/test_connection.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ func testConnection(cmd *cobra.Command, args []string) error {
188188
}
189189
if isolatePluginEnvironment {
190190
sourcePluginConfigs[i].Environment = filterPluginEnv(osEnviron, source.Name, "source")
191+
secretAwareRedactor.AddSecretEnv(sourcePluginConfigs[i].Environment)
191192
}
192193
sourceRegInferred[i] = source.RegistryInferred()
193194
}
@@ -203,6 +204,7 @@ func testConnection(cmd *cobra.Command, args []string) error {
203204
}
204205
if isolatePluginEnvironment {
205206
destinationPluginConfigs[i].Environment = filterPluginEnv(osEnviron, destination.Name, "destination")
207+
secretAwareRedactor.AddSecretEnv(destinationPluginConfigs[i].Environment)
206208
}
207209
destinationRegInferred[i] = destination.RegistryInferred()
208210
}
@@ -219,7 +221,7 @@ func testConnection(cmd *cobra.Command, args []string) error {
219221
PluginRef: ref,
220222
Success: false,
221223
FailureCode: "OTHER",
222-
FailureDescription: err.Error(),
224+
FailureDescription: secretAwareRedactor.RedactStr(err.Error()),
223225
})
224226
}
225227

@@ -350,7 +352,7 @@ func testPluginConnection(ctx context.Context, client plugin.PluginClient, spec
350352
return &testConnectionResult{
351353
Success: false,
352354
FailureCode: "OTHER",
353-
FailureDescription: err.Error(),
355+
FailureDescription: secretAwareRedactor.RedactStr(err.Error()),
354356
}, nil
355357
}
356358

@@ -365,7 +367,7 @@ func testPluginConnection(ctx context.Context, client plugin.PluginClient, spec
365367
return &testConnectionResult{
366368
Success: resp.Success,
367369
FailureCode: resp.FailureCode,
368-
FailureDescription: resp.FailureDescription,
370+
FailureDescription: secretAwareRedactor.RedactStr(resp.FailureDescription),
369371
}, nil
370372
}
371373

‎cli/internal/secrets/secrets.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package secrets
2+
3+
import (
4+
"bytes"
5+
"io"
6+
"strings"
7+
)
8+
9+
type SecretAwareRedactor struct {
10+
secrets map[string]string
11+
}
12+
13+
func NewSecretAwareRedactor() *SecretAwareRedactor {
14+
return &SecretAwareRedactor{secrets: make(map[string]string)}
15+
}
16+
17+
func (s *SecretAwareRedactor) RedactStr(msg string) string {
18+
return string(s.RedactBytes([]byte(msg)))
19+
}
20+
21+
func (s *SecretAwareRedactor) RedactBytes(msg []byte) []byte {
22+
for v, k := range s.secrets {
23+
msg = bytes.ReplaceAll(msg, []byte(v), []byte(k))
24+
}
25+
return msg
26+
}
27+
28+
func (s *SecretAwareRedactor) AddSecretEnv(env []string) {
29+
for _, v := range env {
30+
parts := strings.SplitN(v, "=", 2)
31+
if len(parts) == 2 && len(parts[1]) > 0 {
32+
s.secrets[parts[1]] = parts[0]
33+
}
34+
}
35+
}
36+
37+
type SecretAwareWriter struct {
38+
out io.Writer
39+
redactor *SecretAwareRedactor
40+
}
41+
42+
func NewSecretAwareWriter(out io.Writer, redactor *SecretAwareRedactor) *SecretAwareWriter {
43+
return &SecretAwareWriter{out: out, redactor: redactor}
44+
}
45+
46+
func (s SecretAwareWriter) Write(p []byte) (n int, err error) {
47+
return s.out.Write(s.redactor.RedactBytes(p))
48+
}

‎cli/internal/secrets/secrets_test.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package secrets
2+
3+
import (
4+
"bytes"
5+
"io"
6+
"testing"
7+
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
func TestRedaction(t *testing.T) {
12+
tests := []struct {
13+
name string
14+
msg string
15+
want string
16+
env []string
17+
}{
18+
{
19+
name: "replaces env var value with its name",
20+
env: []string{"DB_PASS=foobar123"},
21+
msg: "wrong password foobar123",
22+
want: "wrong password DB_PASS",
23+
},
24+
{
25+
name: "handles env var value with equals sign",
26+
env: []string{"SECRET=user=foo"},
27+
msg: "wrong password for user=foo",
28+
want: "wrong password for SECRET",
29+
},
30+
{
31+
name: "leaves original msg unchanged",
32+
env: []string{},
33+
msg: "wrong password foobar123",
34+
want: "wrong password foobar123",
35+
},
36+
{
37+
name: "handles env var with empty value",
38+
env: []string{"DB_PASS="},
39+
msg: "wrong password foobar123",
40+
want: "wrong password foobar123",
41+
},
42+
}
43+
for _, tt := range tests {
44+
redactor := NewSecretAwareRedactor()
45+
redactor.AddSecretEnv(tt.env)
46+
t.Run(tt.name, func(t *testing.T) {
47+
got := redactor.RedactStr(tt.msg)
48+
assert.Equal(t, tt.want, got)
49+
})
50+
t.Run(tt.name+" in log", func(t *testing.T) {
51+
out := &bytes.Buffer{}
52+
writer := NewSecretAwareWriter(out, redactor)
53+
_, _ = writer.Write([]byte(tt.msg))
54+
got, _ := io.ReadAll(out)
55+
assert.Equal(t, tt.want, string(got))
56+
})
57+
}
58+
}

0 commit comments

Comments
 (0)