Skip to content

Fix data race in Zerolog console writer#765

Merged
merlimat merged 1 commit intooxia-db:mainfrom
merlimat:fix-zl-data-race
Oct 20, 2025
Merged

Fix data race in Zerolog console writer#765
merlimat merged 1 commit intooxia-db:mainfrom
merlimat:fix-zl-data-race

Conversation

@merlimat
Copy link
Copy Markdown
Collaborator

@merlimat merlimat commented Oct 20, 2025 •
edited
Loading

Seen making many test runs to fail:

==================
WARNING: DATA RACE
Read at 0x000002a0daa8 by goroutine 3221:
  github.com/rs/zerolog.ConsoleWriter.Write()
      /home/runner/go/pkg/mod/github.com/rs/zerolog@v1.34.0/console.go:124 +0x65
  github.com/rs/zerolog.(*ConsoleWriter).Write()
      <autogenerated>:1 +0xd0
  github.com/rs/zerolog.LevelWriterAdapter.WriteLevel()
      /home/runner/go/pkg/mod/github.com/rs/zerolog@v1.34.0/writer.go:27 +0x6f
  github.com/rs/zerolog.(*LevelWriterAdapter).WriteLevel()
      <autogenerated>:1 +0x1f
  github.com/rs/zerolog.(*Event).write()
      /home/runner/go/pkg/mod/github.com/rs/zerolog@v1.34.0/event.go:80 +0x2a1
  github.com/rs/zerolog.(*Event).msg()
      /home/runner/go/pkg/mod/github.com/rs/zerolog@v1.34.0/event.go:151 +0x411
  github.com/rs/zerolog.(*Event).Msg()
      /home/runner/go/pkg/mod/github.com/rs/zerolog@v1.34.0/event.go:110 +0x6a9
  github.com/samber/slog-zerolog/v2.(*ZerologHandler).Handle()
      /home/runner/go/pkg/mod/github.com/samber/slog-zerolog/v2@v2.7.3/handler.go:84 +0x677
  log/slog.(*Logger).log()
      /opt/hostedtoolcache/go/1.25.3/x64/src/log/slog/logger.go:256 +0x24c
  log/slog.(*Logger).Warn()
      /opt/hostedtoolcache/go/1.25.3/x64/src/log/slog/logger.go:219 +0x204
  github.com/oxia-db/oxia/oxia/internal.(*shardManagerImpl).receiveWithRecovery.func2()
      /home/runner/work/oxia/oxia/oxia/internal/shard_manager.go:167 +0x105
  github.com/cenkalti/backoff/v4.doRetryNotify[go.shape.struct {}]()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:107 +0x2d0
  github.com/cenkalti/backoff/v4.RetryNotifyWithTimer.Operation.withEmptyData.func1()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:18 +0x2e
  github.com/cenkalti/backoff/v4.doRetryNotify[go.shape.struct {}]()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:88 +0x1f9
  github.com/cenkalti/backoff/v4.RetryNotifyWithTimer()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:61 +0xa7
  github.com/cenkalti/backoff/v4.RetryNotify()
      /home/runner/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.3.0/retry.go:49 +0x444
  github.com/oxia-db/oxia/oxia/internal.(*shardManagerImpl).receiveWithRecovery()
      /home/runner/work/oxia/oxia/oxia/internal/shard_manager.go:148 +0x41a
  github.com/oxia-db/oxia/oxia/internal.(*shardManagerImpl).receiveWithRecovery-fm()
      <autogenerated>:1 +0x33
  github.com/oxia-db/oxia/common/process.DoWithLabels.func1()
      /home/runner/work/oxia/oxia/common/process/pprof.go:46 +0x41
  runtime/pprof.Do()
      /opt/hostedtoolcache/go/1.25.3/x64/src/runtime/pprof/runtime.go:51 +0x111
  github.com/oxia-db/oxia/common/process.DoWithLabels()
      /home/runner/work/oxia/oxia/common/process/pprof.go:42 +0x3b5
  github.com/oxia-db/oxia/oxia/internal.(*shardManagerImpl).start.gowrap1()
      /home/runner/work/oxia/oxia/oxia/internal/shard_manager.go:92 +0x5d

Previous write at 0x000002a0daa8 by main goroutine:
  testing.runExample.func2()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/run_example.go:55 +0xde
  runtime.deferreturn()
      /opt/hostedtoolcache/go/1.25.3/x64/src/runtime/panic.go:589 +0x5d
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:190 +0x3dd
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:188 +0x3ad
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:316 +0x2bd5
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:106 +0x6da
  github.com/oxia-db/oxia/server/kv.newKVPebble()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:252 +0x1312
  github.com/oxia-db/oxia/server/kv.(*PebbleFactory).NewKV()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:123 +0x44
  github.com/oxia-db/oxia/server/kv.NewDB()
      /home/runner/work/oxia/oxia/server/kv/db.go:99 +0x9a
  github.com/oxia-db/oxia/server.NewLeaderController()
      /home/runner/work/oxia/oxia/server/leader_controller.go:159 +0x971
  github.com/oxia-db/oxia/server.(*shardsDirector).GetOrCreateLeader()
      /home/runner/work/oxia/oxia/server/shards_director.go:149 +0x3ef
  github.com/oxia-db/oxia/server.(*Standalone).initializeShards()
      /home/runner/work/oxia/oxia/server/standalone.go:117 +0x9d
  github.com/oxia-db/oxia/server.NewStandalone()
      /home/runner/work/oxia/oxia/server/standalone.go:89 +0x65c
  github.com/oxia-db/oxia/oxia.initExampleServer()
      /home/runner/work/oxia/oxia/oxia/example_test.go:36 +0xfb
  github.com/oxia-db/oxia/oxia.ExampleNotifications()
      /home/runner/work/oxia/oxia/oxia/example_test.go:125 +0x48
  testing.runExample()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/run_example.go:63 +0x4db
  testing.runExamples()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/example.go:41 +0x214
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:188 +0x3ad
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:316 +0x2bd5
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:106 +0x6da
  github.com/oxia-db/oxia/server/kv.newKVPebble()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:252 +0x1312
  github.com/oxia-db/oxia/server/kv.(*PebbleFactory).NewKV()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:123 +0x44
  github.com/oxia-db/oxia/server/kv.NewDB()
      /home/runner/work/oxia/oxia/server/kv/db.go:99 +0x9a
  github.com/oxia-db/oxia/server.NewLeaderController()
      /home/runner/work/oxia/oxia/server/leader_controller.go:159 +0x971
  github.com/oxia-db/oxia/server.(*shardsDirector).GetOrCreateLeader()
      /home/runner/work/oxia/oxia/server/shards_director.go:149 +0x3ef
  github.com/oxia-db/oxia/server.(*Standalone).initializeShards()
      /home/runner/work/oxia/oxia/server/standalone.go:117 +0x9d
  github.com/oxia-db/oxia/server.NewStandalone()
      /home/runner/work/oxia/oxia/server/standalone.go:89 +0x65c
  github.com/oxia-db/oxia/oxia.initExampleServer()
      /home/runner/work/oxia/oxia/oxia/example_test.go:36 +0xfb
  github.com/oxia-db/oxia/oxia.ExampleAsyncClient()
      /home/runner/work/oxia/oxia/oxia/example_test.go:85 +0x48
  testing.runExample()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/run_example.go:63 +0x4db
  testing.runExamples()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/example.go:41 +0x214
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:188 +0x3ad
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:316 +0x2bd5
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:106 +0x6da
  github.com/oxia-db/oxia/server/kv.newKVPebble()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:252 +0x1312
  github.com/oxia-db/oxia/server/kv.(*PebbleFactory).NewKV()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:123 +0x44
  github.com/oxia-db/oxia/server/kv.NewDB()
      /home/runner/work/oxia/oxia/server/kv/db.go:99 +0x9a
  github.com/oxia-db/oxia/server.NewLeaderController()
      /home/runner/work/oxia/oxia/server/leader_controller.go:159 +0x971
  github.com/oxia-db/oxia/server.(*shardsDirector).GetOrCreateLeader()
      /home/runner/work/oxia/oxia/server/shards_director.go:149 +0x3ef
  github.com/oxia-db/oxia/server.(*Standalone).initializeShards()
      /home/runner/work/oxia/oxia/server/standalone.go:117 +0x9d
  github.com/oxia-db/oxia/server.NewStandalone()
      /home/runner/work/oxia/oxia/server/standalone.go:89 +0x65c
  github.com/oxia-db/oxia/oxia.initExampleServer()
      /home/runner/work/oxia/oxia/oxia/example_test.go:36 +0xfb
  github.com/oxia-db/oxia/oxia.Example()
      /home/runner/work/oxia/oxia/oxia/example_test.go:45 +0x48
  testing.runExample()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/run_example.go:63 +0x4db
  testing.runExamples()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/example.go:41 +0x214
  testing.(*M).Run()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2339 +0x1038
  main.main()
      _testmain.go:129 +0x164

Goroutine 3221 (running) created at:
  github.com/oxia-db/oxia/oxia/internal.(*shardManagerImpl).start()
      /home/runner/work/oxia/oxia/oxia/internal/shard_manager.go:92 +0x24c
  github.com/oxia-db/oxia/oxia/internal.NewShardManager()
      /home/runner/work/oxia/oxia/oxia/internal/shard_manager.go:77 +0x4b6
  github.com/oxia-db/oxia/oxia.NewAsyncClient()
      /home/runner/work/oxia/oxia/oxia/async_client_impl.go:71 +0x36b
  github.com/oxia-db/oxia/oxia.NewSyncClient()
      /home/runner/work/oxia/oxia/oxia/sync_client_impl.go:43 +0x204
  github.com/oxia-db/oxia/oxia.TestSessionEphemeralKeysLeak()
      /home/runner/work/oxia/oxia/oxia/session_test.go:57 +0x6a9
  github.com/oxia-db/oxia/oxia.(*clientImpl).Close()
      /home/runner/work/oxia/oxia/oxia/async_client_impl.go:104 +0xaf
  github.com/oxia-db/oxia/oxia.TestSessionEphemeralKeysLeak()
      /home/runner/work/oxia/oxia/oxia/session_test.go:54 +0x66e
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:540 +0x7403
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:195 +0x537
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:190 +0x3dd
  github.com/cockroachdb/pebble/v2.(*versionSet).create()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/version_set.go:188 +0x3ad
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:316 +0x2bd5
  github.com/cockroachdb/pebble/v2.Open()
      /home/runner/go/pkg/mod/github.com/cockroachdb/pebble/v2@v2.1.0/open.go:106 +0x6da
  github.com/oxia-db/oxia/server/kv.newKVPebble()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:252 +0x1312
  github.com/oxia-db/oxia/server/kv.(*PebbleFactory).NewKV()
      /home/runner/work/oxia/oxia/server/kv/kv_pebble.go:123 +0x44
  github.com/oxia-db/oxia/server/kv.NewDB()
      /home/runner/work/oxia/oxia/server/kv/db.go:99 +0x9a
  github.com/oxia-db/oxia/server.NewLeaderController()
      /home/runner/work/oxia/oxia/server/leader_controller.go:159 +0x971
  github.com/oxia-db/oxia/server.(*shardsDirector).GetOrCreateLeader()
      /home/runner/work/oxia/oxia/server/shards_director.go:149 +0x3ef
  github.com/oxia-db/oxia/server.(*Standalone).initializeShards()
      /home/runner/work/oxia/oxia/server/standalone.go:117 +0x9d
  github.com/oxia-db/oxia/server.NewStandalone()
      /home/runner/work/oxia/oxia/server/standalone.go:89 +0x65c
  github.com/oxia-db/oxia/oxia.TestSessionEphemeralKeysLeak()
      /home/runner/work/oxia/oxia/oxia/session_test.go:31 +0x97
  testing.tRunner()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0x21c
  testing.(*T).Run.gowrap1()
      /opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x44
==================

@merlimat
Fix data race in Zerolog console writer
01a0df9 
Signed-off-by: Matteo Merli <mmerli@apache.org>
@merlimat merlimat merged commit d19ca6d into oxia-db:main Oct 20, 2025
6 checks passed
@merlimat merlimat deleted the fix-zl-data-race branch October 20, 2025 23:49
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@mattisonchao mattisonchao Awaiting requested review from mattisonchao mattisonchao is a code owner

@RobertIndie RobertIndie Awaiting requested review from RobertIndie RobertIndie is a code owner

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

1 participant

@merlimat