kv/kvserver: TestServerSideBoundedStalenessNegotiation failed #71047
Description
kv/kvserver.TestServerSideBoundedStalenessNegotiation failed with artifacts on master @ bc42b6626366da8b2cdd470382dd5fb8202ba0c5:
--- FAIL: TestServerSideBoundedStalenessNegotiation/strict=true/empty_and_intent_key,_min_bound_equal_to_intent_ts,_min_bound_below_closed_ts (0.10s)
=== RUN TestServerSideBoundedStalenessNegotiation/strict=true/empty_and_intent_key,_min_bound_above_intent_ts,_min_bound_above_closed_ts
replica_closedts_internal_test.go:1001:
Error Trace: replica_closedts_internal_test.go:1001
Error: Expect "computing minimum intent timestamp: cannot read undeclared span /Local/Lock/Intent"b"
declared:
read global: a{-\x00} at 0.000000020,5
read global: b{-\x00} at 0.000000020,5
stack:
goroutine 17976 [running]:
runtime/debug.Stack(0xc000cb42e8, 0xbe4f18c, 0x1)
/usr/local/go/src/runtime/debug/stack.go:24 +0xab
github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset.(*SpanSet).checkAllowed(0xc000cb42a0, 0x0, 0xc0005ea150, 0x7, 0x7, 0x0, 0x0, 0x0, 0xc0012eb950, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset/spanset.go:352 +0x35b
github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset.(*SpanSet).CheckAllowedAt(0xc000cb42a0, 0x0, 0xc0005ea150, 0x7, 0x7, 0x0, 0x0, 0x0, 0x14, 0x5, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset/spanset.go:294 +0x12f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset.(*EngineIterator).SeekEngineKeyGE(0xc00054e6f0, 0xc0005ea150, 0x7, 0x7, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/spanset/batch.go:257 +0x326
github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.computeMinIntentTimestamp(0x9d3e418, 0xc001382b40, 0xc0009ab201, 0x1, 0x1, 0xc0005ea102, 0x2, 0x2, 0x1388, 0xf4240, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_resolved_timestamp.go:112 +0x367
github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.QueryResolvedTimestamp(0x9ccd238, 0xc00054e630, 0x9d3e418, 0xc001382b40, 0x9e19f10, 0xc0005d2bd0, 0x14, 0x5, 0x1, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_query_resolved_timestamp.go:74 +0x455
github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand(0x9ccd238, 0xc00054e630, 0x9e18228, 0xc001382b40, 0x9e19f10, 0xc0005d2bd0, 0x0, 0x14, 0x5, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_evaluate.go:507 +0x835
github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch(0x9ccd238, 0xc00054e630, 0x0, 0x0, 0x9e18228, 0xc001382b40, 0x9e19f10, 0xc0005d2bd0, 0x0, 0xc00157e540, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_evaluate.go:271 +0x499
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatchWithServersideRefreshes(0xc001002000, 0x9ccd238, 0xc00054e630, 0x9e18228, 0xc001382b40, 0x9e19f10, 0xc0005d2bd0, 0xc00157e540, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_read.go:291 +0x2b4
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeReadOnlyBatch(0xc001002000, 0x9ccd238, 0xc00054e630, 0xc00157e540, 0xc0005e41e0, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_read.go:85 +0x631
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc001002000, 0x9ccd238, 0xc00054e630, 0xc00157e540, 0x7035c78, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:429 +0x487
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc001002000, 0x9ccd238, 0xc000b96cf0, 0x1, 0xc00157e540, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:155 +0x349
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:95
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc00065e400, 0x9ccd238, 0xc000b96cf0, 0x14, 0x5, 0x1, 0x0, 0x0, 0x0, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:190 +0x5d8
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).executeServerSideBoundedStalenessNegotiation(0xc00065e400, 0x9ccd238, 0xc000b96cf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:411 +0x6d8
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc00065e400, 0x9ccd1c8, 0xc000128010, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:65 +0x19b8
github.com/cockroachdb/cockroach/pkg/kv/kvserver.TestServerSideBoundedStalenessNegotiation.func1.2(0xc0003e6300)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_closedts_internal_test.go:992 +0xda5
testing.tRunner(0xc0003e6300, 0xc0005da000)
/usr/local/go/src/testing/testing.go:1193 +0x203
created by testing.(*T).Run
/usr/local/go/src/testing/testing.go:1238 +0x5d8" to match "bounded staleness read .* could not be satisfied"
Test: TestServerSideBoundedStalenessNegotiation/strict=true/empty_and_intent_key,_min_bound_above_intent_ts,_min_bound_above_closed_ts
--- FAIL: TestServerSideBoundedStalenessNegotiation/strict=true/empty_and_intent_key,_min_bound_above_intent_ts,_min_bound_above_closed_ts (0.10s)
Reproduce
To reproduce, try:
make stressrace TESTS=TestServerSideBoundedStalenessNegotiation PKG=./pkg/kv/kvserver TESTTIMEOUT=5m STRESSFLAGS='-timeout 5m' 2>&1Parameters in this failure:
- GOFLAGS=-json