sql/catalog/lease: TestTxnObeysTableModificationTime failed #107736
Description
sql/catalog/lease.TestTxnObeysTableModificationTime failed with artifacts on master @ 4413ec75b616592066f04d5302e7026fe7fd61e9:
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/internal.go:1746
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql.(*InternalDB).Txn
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/internal.go:1681
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/gcjob.persistProgress
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/gcjob/gc_job_utils.go:281
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/gcjob.schemaChangeGCResumer.deleteDataAndWaitForGC
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/gcjob/gc_job.go:333
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/gcjob.schemaChangeGCResumer.Resume
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/sql/gcjob/gc_job.go:314
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1646
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1647
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:490
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).resumeJob.func2
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:404
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | [...repeated from below...]
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 +Wraps: (6) failed to read query result
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 +Wraps: (7) candidate pg code: 57014
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 +Wraps: (8) attached stack trace
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + -- stack trace:
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/util/cancelchecker.init
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | github.com/cockroachdb/cockroach/pkg/util/cancelchecker/cancel_checker.go:80
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.doInit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:6349
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.doInit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:6326
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.doInit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:6326
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.doInit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:6326
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.doInit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:6326
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.doInit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:6326
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.main
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/proc.go:233
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | runtime.goexit
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 + | GOROOT/src/runtime/asm_arm64.s:1172
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 +Wraps: (9) query execution canceled
W230727 17:24:34.877426 111435 sql/gcjob/gc_job_utils.go:299 [T1,n1,job=SCHEMA CHANGE GC id=886073834508419073] 5757 +Error types: (1) *withstack.withStack (2) *errutil.withPrefix (3) *withstack.withStack (4) *errutil.withPrefix (5) *withstack.withStack (6) *errutil.withPrefix (7) *pgerror.withCandidateCode (8) *withstack.withStack (9) *errutil.leafError
I230727 17:24:34.877863 95429 jobs/registry.go:1606 [T1,n1] 5758 AUTO SPAN CONFIG RECONCILIATION job 886073825006944257: stepping through state succeeded
E230727 17:24:34.878075 94546 server/server_sql.go:521 [T1,n1] 5760 failed to run update of instance with new session ID: node unavailable; try another peer
E230727 17:24:34.878368 95429 jobs/registry.go:1004 [T1,n1] 5761 error getting live session: node unavailable; try another peer
W230727 17:24:34.884762 104545 jobs/adopt.go:534 [T1,n1] 5762 could not clear job claim: clear-job-claim: failed to send RPC: sending to all replicas failed; last error: ba: Scan [/Table/15/1/102,/Table/15/1/103), [txn: 85ba6b9f], [can-forward-ts] RPC error: node unavailable; try another peer
I230727 17:24:34.895317 93874 server/server_controller_orchestration.go:263 [T1,n1] 5763 server controller shutting down ungracefully
I230727 17:24:34.895374 93874 server/server_controller_orchestration.go:274 [T1,n1] 5764 waiting for tenant servers to report stopped
W230727 17:24:34.895477 93874 server/server_sql.go:1685 [T1,n1] 5765 server shutdown without a prior graceful drain
--- FAIL: TestTxnObeysTableModificationTime (3.39s)
Same failure on other branches
- sql/catalog/lease: TestTxnObeysTableModificationTime failed [liveness failure caused jobs failure] #107159 sql/catalog/lease: TestTxnObeysTableModificationTime failed [liveness failure caused jobs failure] [C-test-failure O-robot T-sql-foundations branch-release-23.1]
This test on roachdash | Improve this report!
Jira issue: CRDB-30177